2024-12-09 06:14:54,322 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-12-09 06:14:54,367 main DEBUG Took 0.042476 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 06:14:54,368 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 06:14:54,368 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 06:14:54,369 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 06:14:54,371 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,388 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 06:14:54,409 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,411 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,412 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,412 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,413 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,414 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,415 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,415 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,416 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,417 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,418 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,419 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,420 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,420 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,421 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,422 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,422 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,422 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,423 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 06:14:54,424 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,424 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 06:14:54,429 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 06:14:54,431 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 06:14:54,434 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 06:14:54,435 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 06:14:54,437 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 06:14:54,438 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 06:14:54,449 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 06:14:54,454 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 06:14:54,459 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 06:14:54,460 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 06:14:54,461 main DEBUG createAppenders(={Console}) 2024-12-09 06:14:54,462 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-12-09 06:14:54,462 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-12-09 06:14:54,462 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-12-09 06:14:54,463 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 06:14:54,464 main DEBUG OutputStream closed 2024-12-09 06:14:54,464 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 06:14:54,466 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 06:14:54,467 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-12-09 06:14:54,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 06:14:54,591 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 06:14:54,592 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 06:14:54,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 06:14:54,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 06:14:54,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 06:14:54,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 06:14:54,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 06:14:54,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 06:14:54,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 06:14:54,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 06:14:54,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 06:14:54,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 06:14:54,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 06:14:54,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 06:14:54,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 06:14:54,605 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 06:14:54,607 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 06:14:54,609 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 06:14:54,611 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-12-09 06:14:54,611 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 06:14:54,613 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-12-09T06:14:55,004 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b 2024-12-09 06:14:55,008 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 06:14:55,010 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T06:14:55,022 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-09T06:14:55,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T06:14:55,122 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50, deleteOnExit=true 2024-12-09T06:14:55,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T06:14:55,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/test.cache.data in system properties and HBase conf 2024-12-09T06:14:55,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T06:14:55,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/hadoop.log.dir in system properties and HBase conf 2024-12-09T06:14:55,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T06:14:55,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T06:14:55,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T06:14:55,346 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T06:14:55,536 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T06:14:55,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T06:14:55,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T06:14:55,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T06:14:55,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T06:14:55,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T06:14:55,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T06:14:55,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T06:14:55,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T06:14:55,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T06:14:55,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/nfs.dump.dir in system properties and HBase conf 2024-12-09T06:14:55,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/java.io.tmpdir in system properties and HBase conf 2024-12-09T06:14:55,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T06:14:55,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T06:14:55,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T06:15:24,684 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T06:15:24,790 INFO [Time-limited test {}] log.Log(170): Logging initialized @31566ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T06:15:24,879 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T06:15:24,958 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T06:15:24,992 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T06:15:24,992 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T06:15:24,994 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T06:15:25,022 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T06:15:25,034 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/hadoop.log.dir/,AVAILABLE} 2024-12-09T06:15:25,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T06:15:25,294 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/java.io.tmpdir/jetty-localhost-38811-hadoop-hdfs-3_4_1-tests_jar-_-any-17247125761701721624/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T06:15:25,301 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38811} 2024-12-09T06:15:25,301 INFO [Time-limited test {}] server.Server(415): Started @32078ms 2024-12-09T06:15:26,161 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T06:15:26,173 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T06:15:26,174 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T06:15:26,175 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T06:15:26,175 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T06:15:26,176 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/hadoop.log.dir/,AVAILABLE} 2024-12-09T06:15:26,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T06:15:26,321 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/java.io.tmpdir/jetty-localhost-34087-hadoop-hdfs-3_4_1-tests_jar-_-any-15933548409304933316/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T06:15:26,322 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:34087} 2024-12-09T06:15:26,322 INFO [Time-limited test {}] server.Server(415): Started @33099ms 2024-12-09T06:15:26,394 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T06:15:27,922 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/data/data1/current/BP-519108604-172.17.0.3-1733724896367/current, will proceed with Du for space computation calculation, 2024-12-09T06:15:27,925 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/data/data2/current/BP-519108604-172.17.0.3-1733724896367/current, will proceed with Du for space computation calculation, 2024-12-09T06:15:28,058 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T06:15:28,177 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa929a40def87125a with lease ID 0x4d3a16c7e0d73de8: Processing first storage report for DS-3fd2b4fb-422a-424f-926f-99af4e7c082d from datanode DatanodeRegistration(127.0.0.1:40437, datanodeUuid=b99168bb-add6-453c-b13d-a80faae4a1f5, infoPort=38883, infoSecurePort=0, ipcPort=38241, storageInfo=lv=-57;cid=testClusterID;nsid=1060471091;c=1733724896367) 2024-12-09T06:15:28,178 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa929a40def87125a with lease ID 0x4d3a16c7e0d73de8: from storage DS-3fd2b4fb-422a-424f-926f-99af4e7c082d node DatanodeRegistration(127.0.0.1:40437, datanodeUuid=b99168bb-add6-453c-b13d-a80faae4a1f5, infoPort=38883, infoSecurePort=0, ipcPort=38241, storageInfo=lv=-57;cid=testClusterID;nsid=1060471091;c=1733724896367), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T06:15:28,179 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa929a40def87125a with lease ID 0x4d3a16c7e0d73de8: Processing first storage report for DS-13c4b918-d35c-4cb7-99ff-349ed6f10bfe from datanode DatanodeRegistration(127.0.0.1:40437, datanodeUuid=b99168bb-add6-453c-b13d-a80faae4a1f5, infoPort=38883, infoSecurePort=0, ipcPort=38241, storageInfo=lv=-57;cid=testClusterID;nsid=1060471091;c=1733724896367) 2024-12-09T06:15:28,179 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa929a40def87125a with lease ID 0x4d3a16c7e0d73de8: from storage DS-13c4b918-d35c-4cb7-99ff-349ed6f10bfe node DatanodeRegistration(127.0.0.1:40437, datanodeUuid=b99168bb-add6-453c-b13d-a80faae4a1f5, infoPort=38883, infoSecurePort=0, ipcPort=38241, storageInfo=lv=-57;cid=testClusterID;nsid=1060471091;c=1733724896367), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T06:15:28,222 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b 2024-12-09T06:15:28,403 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/zookeeper_0, clientPort=56248, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T06:15:28,430 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56248 2024-12-09T06:15:28,455 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T06:15:28,459 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T06:15:28,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741825_1001 (size=7) 2024-12-09T06:15:29,255 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d with version=8 2024-12-09T06:15:29,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/hbase-staging 2024-12-09T06:15:29,351 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T06:15:29,629 INFO [Time-limited test {}] client.ConnectionUtils(128): master/82ac79f0323b:0 server-side Connection retries=45 2024-12-09T06:15:29,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T06:15:29,642 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T06:15:29,649 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T06:15:29,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T06:15:29,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T06:15:29,815 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T06:15:29,886 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T06:15:29,902 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T06:15:29,908 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T06:15:29,940 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5458 (auto-detected) 2024-12-09T06:15:29,941 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-09T06:15:29,965 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43241 2024-12-09T06:15:29,993 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43241 connecting to ZooKeeper ensemble=127.0.0.1:56248 2024-12-09T06:15:30,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432410x0, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T06:15:30,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43241-0x100093700330000 connected 2024-12-09T06:15:30,331 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T06:15:30,343 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T06:15:30,370 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T06:15:30,375 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d, hbase.cluster.distributed=false 2024-12-09T06:15:30,440 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T06:15:30,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43241 2024-12-09T06:15:30,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43241 2024-12-09T06:15:30,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43241 2024-12-09T06:15:30,487 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43241 2024-12-09T06:15:30,487 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43241 2024-12-09T06:15:30,646 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/82ac79f0323b:0 server-side Connection retries=45 2024-12-09T06:15:30,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T06:15:30,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T06:15:30,649 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T06:15:30,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T06:15:30,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T06:15:30,654 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T06:15:30,658 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T06:15:30,662 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42797 2024-12-09T06:15:30,665 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42797 connecting to ZooKeeper ensemble=127.0.0.1:56248 2024-12-09T06:15:30,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T06:15:30,674 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T06:15:30,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427970x0, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T06:15:30,775 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42797-0x100093700330001 connected 2024-12-09T06:15:30,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T06:15:30,794 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T06:15:30,808 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T06:15:30,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T06:15:30,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T06:15:30,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42797 2024-12-09T06:15:30,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42797 2024-12-09T06:15:30,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42797 2024-12-09T06:15:30,839 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42797 2024-12-09T06:15:30,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42797 2024-12-09T06:15:30,863 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;82ac79f0323b:43241 2024-12-09T06:15:30,869 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/82ac79f0323b,43241,1733724929416 2024-12-09T06:15:30,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T06:15:30,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T06:15:30,921 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/82ac79f0323b,43241,1733724929416 2024-12-09T06:15:30,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T06:15:30,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:30,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:30,977 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T06:15:30,979 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/82ac79f0323b,43241,1733724929416 from backup master directory 2024-12-09T06:15:31,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T06:15:31,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/82ac79f0323b,43241,1733724929416 2024-12-09T06:15:31,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T06:15:31,078 WARN [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T06:15:31,078 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=82ac79f0323b,43241,1733724929416 2024-12-09T06:15:31,080 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T06:15:31,084 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T06:15:31,171 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/hbase.id] with ID: 14f39449-667b-4aed-944a-0c1e2e63b424 2024-12-09T06:15:31,171 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/.tmp/hbase.id 2024-12-09T06:15:31,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741826_1002 (size=42) 2024-12-09T06:15:31,221 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/.tmp/hbase.id]:[hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/hbase.id] 2024-12-09T06:15:31,282 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T06:15:31,288 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T06:15:31,313 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-12-09T06:15:31,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:31,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741827_1003 (size=196) 2024-12-09T06:15:31,439 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:15:31,447 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T06:15:31,471 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:31,478 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T06:15:31,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741828_1004 (size=1189) 2024-12-09T06:15:31,966 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store 2024-12-09T06:15:31,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741829_1005 (size=34) 2024-12-09T06:15:32,412 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T06:15:32,431 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:15:32,434 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T06:15:32,435 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T06:15:32,435 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T06:15:32,437 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T06:15:32,438 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T06:15:32,438 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T06:15:32,440 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733724932434Disabling compacts and flushes for region at 1733724932434Disabling writes for close at 1733724932437 (+3 ms)Writing region close event to WAL at 1733724932438 (+1 ms)Closed at 1733724932438 2024-12-09T06:15:32,446 WARN [master/82ac79f0323b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/.initializing 2024-12-09T06:15:32,446 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/WALs/82ac79f0323b,43241,1733724929416 2024-12-09T06:15:32,462 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T06:15:32,486 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82ac79f0323b%2C43241%2C1733724929416, suffix=, logDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/WALs/82ac79f0323b,43241,1733724929416, archiveDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/oldWALs, maxLogs=10 2024-12-09T06:15:32,520 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/WALs/82ac79f0323b,43241,1733724929416/82ac79f0323b%2C43241%2C1733724929416.1733724932492, exclude list is [], retry=0 2024-12-09T06:15:32,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-3fd2b4fb-422a-424f-926f-99af4e7c082d,DISK] 2024-12-09T06:15:32,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T06:15:32,610 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/WALs/82ac79f0323b,43241,1733724929416/82ac79f0323b%2C43241%2C1733724929416.1733724932492 2024-12-09T06:15:32,614 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38883:38883)] 2024-12-09T06:15:32,615 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:15:32,616 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:15:32,622 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,625 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T06:15:32,760 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:32,764 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:32,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,774 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T06:15:32,775 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:32,778 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:15:32,778 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,786 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T06:15:32,787 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:32,790 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:15:32,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T06:15:32,798 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:32,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:15:32,802 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,808 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,811 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,821 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,822 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,832 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T06:15:32,840 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T06:15:32,869 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:15:32,874 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61373823, jitterRate=-0.08545877039432526}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T06:15:32,886 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733724932649Initializing all the Stores at 1733724932657 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733724932658 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724932661 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724932661Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724932661Cleaning up temporary data from old regions at 1733724932823 (+162 ms)Region opened successfully at 1733724932886 (+63 ms) 2024-12-09T06:15:32,889 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T06:15:33,020 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ff6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82ac79f0323b/172.17.0.3:0 2024-12-09T06:15:33,082 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T06:15:33,105 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T06:15:33,106 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T06:15:33,113 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T06:15:33,118 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 4 msec 2024-12-09T06:15:33,128 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-12-09T06:15:33,129 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T06:15:33,195 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T06:15:33,225 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T06:15:33,242 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T06:15:33,247 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T06:15:33,260 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T06:15:33,281 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T06:15:33,286 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T06:15:33,304 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T06:15:33,324 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T06:15:33,328 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T06:15:33,351 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T06:15:33,424 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T06:15:33,441 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T06:15:33,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T06:15:33,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:33,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T06:15:33,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:33,479 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=82ac79f0323b,43241,1733724929416, sessionid=0x100093700330000, setting cluster-up flag (Was=false) 2024-12-09T06:15:33,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:33,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:33,617 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T06:15:33,635 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82ac79f0323b,43241,1733724929416 2024-12-09T06:15:33,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:33,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:33,734 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T06:15:33,745 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=82ac79f0323b,43241,1733724929416 2024-12-09T06:15:33,779 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T06:15:33,874 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(746): ClusterId : 14f39449-667b-4aed-944a-0c1e2e63b424 2024-12-09T06:15:33,878 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T06:15:33,915 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T06:15:33,916 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T06:15:33,946 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T06:15:33,953 DEBUG [RS:0;82ac79f0323b:42797 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271acf2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=82ac79f0323b/172.17.0.3:0 2024-12-09T06:15:34,009 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;82ac79f0323b:42797 2024-12-09T06:15:34,014 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T06:15:34,014 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T06:15:34,015 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T06:15:34,033 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(2659): reportForDuty to master=82ac79f0323b,43241,1733724929416 with port=42797, startcode=1733724930590 2024-12-09T06:15:34,041 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T06:15:34,065 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T06:15:34,093 DEBUG [RS:0;82ac79f0323b:42797 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T06:15:34,109 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T06:15:34,167 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 82ac79f0323b,43241,1733724929416 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T06:15:34,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42745, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T06:15:34,226 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43241 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:34,249 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/82ac79f0323b:0, corePoolSize=5, maxPoolSize=5 2024-12-09T06:15:34,250 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/82ac79f0323b:0, corePoolSize=5, maxPoolSize=5 2024-12-09T06:15:34,250 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/82ac79f0323b:0, corePoolSize=5, maxPoolSize=5 2024-12-09T06:15:34,258 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/82ac79f0323b:0, corePoolSize=5, maxPoolSize=5 2024-12-09T06:15:34,259 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/82ac79f0323b:0, corePoolSize=10, maxPoolSize=10 2024-12-09T06:15:34,259 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,260 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/82ac79f0323b:0, corePoolSize=2, maxPoolSize=2 2024-12-09T06:15:34,260 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,293 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T06:15:34,294 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T06:15:34,307 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T06:15:34,308 WARN [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T06:15:34,309 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733724964309 2024-12-09T06:15:34,312 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T06:15:34,313 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T06:15:34,319 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:34,321 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T06:15:34,322 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T06:15:34,323 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T06:15:34,323 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T06:15:34,326 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T06:15:34,339 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,353 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T06:15:34,355 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T06:15:34,356 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T06:15:34,364 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T06:15:34,365 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T06:15:34,381 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/82ac79f0323b:0:becomeActiveMaster-HFileCleaner.large.0-1733724934367,5,FailOnTimeoutGroup] 2024-12-09T06:15:34,390 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/82ac79f0323b:0:becomeActiveMaster-HFileCleaner.small.0-1733724934382,5,FailOnTimeoutGroup] 2024-12-09T06:15:34,391 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,391 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T06:15:34,398 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,399 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741831_1007 (size=1321) 2024-12-09T06:15:34,418 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(2659): reportForDuty to master=82ac79f0323b,43241,1733724929416 with port=42797, startcode=1733724930590 2024-12-09T06:15:34,421 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T06:15:34,425 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43241 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 82ac79f0323b,42797,1733724930590 2024-12-09T06:15:34,425 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d 2024-12-09T06:15:34,428 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43241 {}] master.ServerManager(517): Registering regionserver=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:34,445 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d 2024-12-09T06:15:34,445 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36419 2024-12-09T06:15:34,446 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T06:15:34,473 DEBUG [RS:0;82ac79f0323b:42797 {}] zookeeper.ZKUtil(111): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/82ac79f0323b,42797,1733724930590 2024-12-09T06:15:34,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T06:15:34,473 WARN [RS:0;82ac79f0323b:42797 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T06:15:34,474 INFO [RS:0;82ac79f0323b:42797 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T06:15:34,474 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/WALs/82ac79f0323b,42797,1733724930590 2024-12-09T06:15:34,486 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [82ac79f0323b,42797,1733724930590] 2024-12-09T06:15:34,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741832_1008 (size=32) 2024-12-09T06:15:34,555 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T06:15:34,594 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T06:15:34,605 INFO [RS:0;82ac79f0323b:42797 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T06:15:34,605 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,610 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T06:15:34,633 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T06:15:34,636 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,637 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,642 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,644 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,644 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,645 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,646 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/82ac79f0323b:0, corePoolSize=2, maxPoolSize=2 2024-12-09T06:15:34,647 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,649 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,649 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,650 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,651 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,653 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/82ac79f0323b:0, corePoolSize=1, maxPoolSize=1 2024-12-09T06:15:34,654 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/82ac79f0323b:0, corePoolSize=3, maxPoolSize=3 2024-12-09T06:15:34,655 DEBUG [RS:0;82ac79f0323b:42797 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0, corePoolSize=3, maxPoolSize=3 2024-12-09T06:15:34,661 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,661 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,664 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,664 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,664 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,664 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=82ac79f0323b,42797,1733724930590-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T06:15:34,720 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T06:15:34,723 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=82ac79f0323b,42797,1733724930590-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,724 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,725 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.Replication(171): 82ac79f0323b,42797,1733724930590 started 2024-12-09T06:15:34,756 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:34,757 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1482): Serving as 82ac79f0323b,42797,1733724930590, RpcServer on 82ac79f0323b/172.17.0.3:42797, sessionid=0x100093700330001 2024-12-09T06:15:34,758 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T06:15:34,758 DEBUG [RS:0;82ac79f0323b:42797 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 82ac79f0323b,42797,1733724930590 2024-12-09T06:15:34,759 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82ac79f0323b,42797,1733724930590' 2024-12-09T06:15:34,759 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T06:15:34,773 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T06:15:34,796 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T06:15:34,796 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T06:15:34,797 DEBUG [RS:0;82ac79f0323b:42797 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 82ac79f0323b,42797,1733724930590 2024-12-09T06:15:34,797 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '82ac79f0323b,42797,1733724930590' 2024-12-09T06:15:34,797 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T06:15:34,809 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T06:15:34,814 DEBUG [RS:0;82ac79f0323b:42797 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T06:15:34,814 INFO [RS:0;82ac79f0323b:42797 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T06:15:34,814 INFO [RS:0;82ac79f0323b:42797 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T06:15:34,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:15:34,923 INFO [RS:0;82ac79f0323b:42797 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T06:15:34,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T06:15:34,941 INFO [RS:0;82ac79f0323b:42797 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82ac79f0323b%2C42797%2C1733724930590, suffix=, logDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/WALs/82ac79f0323b,42797,1733724930590, archiveDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/oldWALs, maxLogs=32 2024-12-09T06:15:34,954 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T06:15:34,954 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:34,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:34,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T06:15:34,965 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T06:15:34,965 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:34,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:34,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T06:15:34,983 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T06:15:34,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:34,988 DEBUG [RS:0;82ac79f0323b:42797 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/WALs/82ac79f0323b,42797,1733724930590/82ac79f0323b%2C42797%2C1733724930590.1733724934947, exclude list is [], retry=0 2024-12-09T06:15:34,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:35,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T06:15:35,014 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T06:15:35,014 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:35,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-3fd2b4fb-422a-424f-926f-99af4e7c082d,DISK] 2024-12-09T06:15:35,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:35,021 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T06:15:35,027 INFO [RS:0;82ac79f0323b:42797 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/WALs/82ac79f0323b,42797,1733724930590/82ac79f0323b%2C42797%2C1733724930590.1733724934947 2024-12-09T06:15:35,029 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740 2024-12-09T06:15:35,035 DEBUG [RS:0;82ac79f0323b:42797 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38883:38883)] 2024-12-09T06:15:35,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740 2024-12-09T06:15:35,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T06:15:35,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T06:15:35,061 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:15:35,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T06:15:35,090 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:15:35,094 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59997493, jitterRate=-0.1059676855802536}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:15:35,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733724934902Initializing all the Stores at 1733724934921 (+19 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733724934921Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733724934929 (+8 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724934929Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733724934929Cleaning up temporary data from old regions at 1733724935054 (+125 ms)Region opened successfully at 1733724935098 (+44 ms) 2024-12-09T06:15:35,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T06:15:35,103 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T06:15:35,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T06:15:35,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T06:15:35,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T06:15:35,107 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T06:15:35,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733724935102Disabling compacts and flushes for region at 1733724935102Disabling writes for close at 1733724935103 (+1 ms)Writing region close event to WAL at 1733724935106 (+3 ms)Closed at 1733724935107 (+1 ms) 2024-12-09T06:15:35,124 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T06:15:35,124 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T06:15:35,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T06:15:35,149 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T06:15:35,157 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T06:15:35,311 DEBUG [82ac79f0323b:43241 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T06:15:35,338 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:35,350 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82ac79f0323b,42797,1733724930590, state=OPENING 2024-12-09T06:15:35,460 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T06:15:35,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:35,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:15:35,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T06:15:35,479 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T06:15:35,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T06:15:35,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:15:35,734 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T06:15:35,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46807, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T06:15:35,783 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T06:15:35,784 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T06:15:35,789 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T06:15:35,805 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=82ac79f0323b%2C42797%2C1733724930590.meta, suffix=.meta, logDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/WALs/82ac79f0323b,42797,1733724930590, archiveDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/oldWALs, maxLogs=32 2024-12-09T06:15:35,835 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/WALs/82ac79f0323b,42797,1733724930590/82ac79f0323b%2C42797%2C1733724930590.meta.1733724935807.meta, exclude list is [], retry=0 2024-12-09T06:15:35,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-3fd2b4fb-422a-424f-926f-99af4e7c082d,DISK] 2024-12-09T06:15:35,867 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/WALs/82ac79f0323b,42797,1733724930590/82ac79f0323b%2C42797%2C1733724930590.meta.1733724935807.meta 2024-12-09T06:15:35,868 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38883:38883)] 2024-12-09T06:15:35,869 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:15:35,873 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T06:15:35,877 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T06:15:35,886 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T06:15:35,902 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T06:15:35,905 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:15:35,906 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T06:15:35,906 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T06:15:35,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T06:15:35,918 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T06:15:35,918 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:35,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:35,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T06:15:35,929 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T06:15:35,929 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:35,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:35,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T06:15:35,937 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T06:15:35,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:35,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:35,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T06:15:35,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T06:15:35,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:35,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T06:15:35,950 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T06:15:35,952 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740 2024-12-09T06:15:35,963 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740 2024-12-09T06:15:35,967 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T06:15:35,967 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T06:15:35,969 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:15:35,995 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T06:15:35,999 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66382904, jitterRate=-0.010817646980285645}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:15:35,999 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T06:15:36,001 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733724935906Writing region info on filesystem at 1733724935907 (+1 ms)Initializing all the Stores at 1733724935909 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733724935909Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733724935913 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724935913Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733724935913Cleaning up temporary data from old regions at 1733724935967 (+54 ms)Running coprocessor post-open hooks at 1733724936000 (+33 ms)Region opened successfully at 1733724936001 (+1 ms) 2024-12-09T06:15:36,021 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733724935689 2024-12-09T06:15:36,058 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:36,061 DEBUG [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T06:15:36,061 INFO [RS_OPEN_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T06:15:36,065 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 82ac79f0323b,42797,1733724930590, state=OPEN 2024-12-09T06:15:38,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T06:15:38,694 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T06:15:38,694 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:38,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T06:15:38,696 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T06:15:38,834 INFO [AsyncFSWAL-0-hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData-prefix:82ac79f0323b,43241,1733724929416 {}] wal.AbstractFSWAL(1368): Slow sync cost: 137 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40437,DS-3fd2b4fb-422a-424f-926f-99af4e7c082d,DISK]] 2024-12-09T06:15:38,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T06:15:38,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=82ac79f0323b,42797,1733724930590 in 3.2120 sec 2024-12-09T06:15:38,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T06:15:38,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 3.7090 sec 2024-12-09T06:15:38,859 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T06:15:38,859 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T06:15:38,898 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:38,900 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:38,937 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:38,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47059, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:38,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 5.1120 sec 2024-12-09T06:15:38,977 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733724938976, completionTime=-1 2024-12-09T06:15:38,980 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T06:15:38,980 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T06:15:39,015 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T06:15:39,015 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733724999015 2024-12-09T06:15:39,015 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733725059015 2024-12-09T06:15:39,015 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 35 msec 2024-12-09T06:15:39,018 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82ac79f0323b,43241,1733724929416-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:39,018 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82ac79f0323b,43241,1733724929416-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:39,019 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82ac79f0323b,43241,1733724929416-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:39,020 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-82ac79f0323b:43241, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:39,021 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:39,025 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:39,058 DEBUG [master/82ac79f0323b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T06:15:39,110 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 8.031sec 2024-12-09T06:15:39,112 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T06:15:39,114 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T06:15:39,115 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T06:15:39,115 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T06:15:39,115 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T06:15:39,116 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82ac79f0323b,43241,1733724929416-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T06:15:39,117 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82ac79f0323b,43241,1733724929416-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T06:15:39,126 DEBUG [master/82ac79f0323b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T06:15:39,127 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T06:15:39,127 INFO [master/82ac79f0323b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=82ac79f0323b,43241,1733724929416-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T06:15:39,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dfad15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:39,214 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T06:15:39,214 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T06:15:39,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:39,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:39,232 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76127ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:39,242 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:39,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b5c665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:39,247 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:39,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:39,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:39,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=82ac79f0323b,43241,1733724929416 2024-12-09T06:15:39,302 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=209, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=902, ProcessCount=11, AvailableMemoryMB=2453 2024-12-09T06:15:39,313 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T06:15:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 82ac79f0323b,43241,1733724929416 2024-12-09T06:15:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1703c4f5 2024-12-09T06:15:39,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T06:15:39,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T06:15:39,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:15:39,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:15:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T06:15:39,386 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T06:15:39,402 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:39,405 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T06:15:39,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-09T06:15:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T06:15:39,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741835_1011 (size=1061) 2024-12-09T06:15:39,493 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d 2024-12-09T06:15:39,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741836_1012 (size=53) 2024-12-09T06:15:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T06:15:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T06:15:39,881 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T06:15:39,882 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T06:15:39,891 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T06:15:39,892 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T06:15:39,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T06:15:39,892 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T06:15:39,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:15:39,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 89f73a4239a4d6e12de44ad55db0a5e8, disabling compactions & flushes 2024-12-09T06:15:39,918 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:39,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:39,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. after waiting 0 ms 2024-12-09T06:15:39,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:39,918 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:39,919 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 89f73a4239a4d6e12de44ad55db0a5e8: Waiting for close lock at 1733724939918Disabling compacts and flushes for region at 1733724939918Disabling writes for close at 1733724939918Writing region close event to WAL at 1733724939918Closed at 1733724939918 2024-12-09T06:15:39,921 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T06:15:39,927 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733724939921"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733724939921"}]},"ts":"1733724939921"} 2024-12-09T06:15:39,931 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T06:15:39,933 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T06:15:39,936 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724939934"}]},"ts":"1733724939934"} 2024-12-09T06:15:39,940 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T06:15:39,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, ASSIGN}] 2024-12-09T06:15:39,945 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, ASSIGN 2024-12-09T06:15:39,947 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, ASSIGN; state=OFFLINE, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=false 2024-12-09T06:15:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T06:15:40,098 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=89f73a4239a4d6e12de44ad55db0a5e8, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:40,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, ASSIGN because future has completed 2024-12-09T06:15:40,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:15:40,273 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:40,273 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:15:40,274 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,274 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:15:40,274 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,274 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,301 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,311 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:15:40,312 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f73a4239a4d6e12de44ad55db0a5e8 columnFamilyName A 2024-12-09T06:15:40,312 DEBUG [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:40,315 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.HStore(327): Store=89f73a4239a4d6e12de44ad55db0a5e8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:15:40,315 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,325 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:15:40,326 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f73a4239a4d6e12de44ad55db0a5e8 columnFamilyName B 2024-12-09T06:15:40,326 DEBUG [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:40,336 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.HStore(327): Store=89f73a4239a4d6e12de44ad55db0a5e8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:15:40,337 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,349 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:15:40,350 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f73a4239a4d6e12de44ad55db0a5e8 columnFamilyName C 2024-12-09T06:15:40,350 DEBUG [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:15:40,353 INFO [StoreOpener-89f73a4239a4d6e12de44ad55db0a5e8-1 {}] regionserver.HStore(327): Store=89f73a4239a4d6e12de44ad55db0a5e8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:15:40,354 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:40,354 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,357 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,359 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,364 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,364 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,366 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:15:40,375 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,389 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:15:40,394 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 89f73a4239a4d6e12de44ad55db0a5e8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68602025, jitterRate=0.02224983274936676}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:15:40,394 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:40,395 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 89f73a4239a4d6e12de44ad55db0a5e8: Running coprocessor pre-open hook at 1733724940274Writing region info on filesystem at 1733724940274Initializing all the Stores at 1733724940281 (+7 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724940281Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724940281Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724940281Cleaning up temporary data from old regions at 1733724940364 (+83 ms)Running coprocessor post-open hooks at 1733724940394 (+30 ms)Region opened successfully at 1733724940395 (+1 ms) 2024-12-09T06:15:40,401 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., pid=6, masterSystemTime=1733724940262 2024-12-09T06:15:40,411 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=89f73a4239a4d6e12de44ad55db0a5e8, regionState=OPEN, openSeqNum=2, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:40,419 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:40,419 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:40,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:15:40,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T06:15:40,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 in 332 msec 2024-12-09T06:15:40,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T06:15:40,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, ASSIGN in 513 msec 2024-12-09T06:15:40,476 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T06:15:40,476 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724940476"}]},"ts":"1733724940476"} 2024-12-09T06:15:40,493 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T06:15:40,502 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T06:15:40,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1400 sec 2024-12-09T06:15:40,554 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T06:15:40,562 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T06:15:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T06:15:40,574 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:40,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e68a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,591 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44d3aff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,597 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143214a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,601 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:40,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:40,614 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T06:15:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 82ac79f0323b,43241,1733724929416 2024-12-09T06:15:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6412aa15 2024-12-09T06:15:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T06:15:40,628 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T06:15:40,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,653 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@372607cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,662 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8ba2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a26ed24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,711 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63fbedff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,744 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea30c3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b3d420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,782 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3338dc15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,805 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586d2fdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,820 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f1f744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,826 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68bfa07a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,843 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b50ba64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,853 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138b395b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54916543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6804f6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,891 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9ecba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,894 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a49bfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:15:40,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:15:40,904 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:15:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:15:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:15:40,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc95064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:15:40,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:15:40,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:15:40,910 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:15:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9b44b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:15:40,928 DEBUG [Thread-144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:40,937 DEBUG [Thread-157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:40,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:40,939 DEBUG [Thread-157 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:40,940 DEBUG [Thread-157 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:40,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:40,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-09T06:15:40,958 DEBUG [Thread-150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:40,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:40,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T06:15:40,967 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:40,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:40,982 DEBUG [Thread-148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:40,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:40,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:40,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:41,012 DEBUG [Thread-161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:41,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:41,030 DEBUG [Thread-152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:41,032 DEBUG [Thread-155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:41,049 DEBUG [Thread-159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:41,062 DEBUG [Thread-146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:15:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:15:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:15:41,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T06:15:41,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:41,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:41,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:15:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:15:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:41,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:15:41,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:41,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:41,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:41,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:41,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:41,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:41,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:41,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:41,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T06:15:41,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724941270, max timestamp=1733724941354Num uniques 859; 2024-12-09T06:15:41,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:15:41,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724941318, max timestamp=1733724941354Num uniques 859; 2024-12-09T06:15:41,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:15:41,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:41,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733724941318, max timestamp=1733724941356Num uniques 859; 2024-12-09T06:15:41,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:15:41,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:41,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/aa44851b3fcb4567bded30e4c0754086 is 50, key is test_row_0/A:col10/1733724941110/Put/seqid=0 2024-12-09T06:15:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,468 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T06:15:41,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741837_1013 (size=14341) 2024-12-09T06:15:41,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/aa44851b3fcb4567bded30e4c0754086 2024-12-09T06:15:41,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:41,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725001530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725001545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725001533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725001550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T06:15:41,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725001545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/7e82557b40104e538de666e2e04e6ad0 is 50, key is test_row_0/B:col10/1733724941110/Put/seqid=0 2024-12-09T06:15:41,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725001688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725001689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725001698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725001701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741838_1014 (size=12001) 2024-12-09T06:15:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/7e82557b40104e538de666e2e04e6ad0 2024-12-09T06:15:41,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725001717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:41,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/92bb28539f8a43ce9f87976254bda142 is 50, key is test_row_0/C:col10/1733724941110/Put/seqid=0 2024-12-09T06:15:41,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741839_1015 (size=12001) 2024-12-09T06:15:41,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/92bb28539f8a43ce9f87976254bda142 2024-12-09T06:15:41,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/aa44851b3fcb4567bded30e4c0754086 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/aa44851b3fcb4567bded30e4c0754086 2024-12-09T06:15:41,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725001908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:41,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:41,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:41,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725001917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725001930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725001938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725001947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:41,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/aa44851b3fcb4567bded30e4c0754086, entries=200, sequenceid=11, filesize=14.0 K 2024-12-09T06:15:42,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/7e82557b40104e538de666e2e04e6ad0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/7e82557b40104e538de666e2e04e6ad0 2024-12-09T06:15:42,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/7e82557b40104e538de666e2e04e6ad0, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T06:15:42,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/92bb28539f8a43ce9f87976254bda142 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/92bb28539f8a43ce9f87976254bda142 2024-12-09T06:15:42,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/92bb28539f8a43ce9f87976254bda142, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T06:15:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T06:15:42,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139285 for 89f73a4239a4d6e12de44ad55db0a5e8 in 964ms, sequenceid=11, compaction requested=false 2024-12-09T06:15:42,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:42,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724941441, max timestamp=1733724942245Num uniques 1068; 2024-12-09T06:15:42,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-09T06:15:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:42,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=109.18 KB heapSize=311.21 KB 2024-12-09T06:15:42,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724941441, max timestamp=1733724942245Num uniques 1018; 2024-12-09T06:15:42,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:15:42,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:42,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:42,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:42,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724941441, max timestamp=1733724942245Num uniques 1009; 2024-12-09T06:15:42,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T06:15:42,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:42,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:42,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:42,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:42,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:42,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/0ea69aba66804923b80beb2b22c00221 is 50, key is test_row_0/A:col10/1733724942273/Put/seqid=0 2024-12-09T06:15:42,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741840_1016 (size=12001) 2024-12-09T06:15:42,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733724942364, max timestamp=1733724942437Num uniques 859; 2024-12-09T06:15:42,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:15:42,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724942364, max timestamp=1733724942446Num uniques 859; 2024-12-09T06:15:42,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:15:42,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724942364, max timestamp=1733724942437Num uniques 859; 2024-12-09T06:15:42,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:15:42,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733724942539, max timestamp=1733724942566Num uniques 1009; 2024-12-09T06:15:42,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-12-09T06:15:42,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724942546, max timestamp=1733724942566Num uniques 1018; 2024-12-09T06:15:42,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-09T06:15:42,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724942546, max timestamp=1733724942566Num uniques 1009; 2024-12-09T06:15:42,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-09T06:15:42,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:42,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:42,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733724942631, max timestamp=1733724942691Num uniques 1009; 2024-12-09T06:15:42,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=19552, suffix heap size=132904,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=427 2024-12-09T06:15:42,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733724942631, max timestamp=1733724942691Num uniques 1077; 2024-12-09T06:15:42,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=19552, suffix heap size=139976,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=427 2024-12-09T06:15:42,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=427, cellSize=19552, totalHeapSize=44648, min timestamp=1733724942631, max timestamp=1733724942691Num uniques 1009; 2024-12-09T06:15:42,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=19552, suffix heap size=132904,new segment heap size=44648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=427 2024-12-09T06:15:42,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/0ea69aba66804923b80beb2b22c00221 2024-12-09T06:15:42,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:42,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725002775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725002777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725002779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725002792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725002805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a8407467cb284a409d07c790479085b6 is 50, key is test_row_0/B:col10/1733724942273/Put/seqid=0 2024-12-09T06:15:42,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741841_1017 (size=12001) 2024-12-09T06:15:42,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725002908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725002917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725002919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725002929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:42,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725002930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:42,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:42,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:42,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:42,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T06:15:43,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725003127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725003137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725003147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725003147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725003149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:43,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:43,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725003437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725003457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725003470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725003469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725003470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a8407467cb284a409d07c790479085b6 2024-12-09T06:15:43,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:43,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/752035daae394490808b33b382c82d10 is 50, key is test_row_0/C:col10/1733724942273/Put/seqid=0 2024-12-09T06:15:43,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741842_1018 (size=12001) 2024-12-09T06:15:43,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:43,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:43,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:43,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725003961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725003966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725003987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725003987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:43,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725003987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:44,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:44,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:44,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:44,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:44,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:44,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:44,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/752035daae394490808b33b382c82d10 2024-12-09T06:15:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/0ea69aba66804923b80beb2b22c00221 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0ea69aba66804923b80beb2b22c00221 2024-12-09T06:15:44,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0ea69aba66804923b80beb2b22c00221, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T06:15:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a8407467cb284a409d07c790479085b6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a8407467cb284a409d07c790479085b6 2024-12-09T06:15:44,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a8407467cb284a409d07c790479085b6, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T06:15:44,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/752035daae394490808b33b382c82d10 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/752035daae394490808b33b382c82d10 2024-12-09T06:15:44,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:44,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:44,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:44,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:44,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/752035daae394490808b33b382c82d10, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T06:15:44,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=154.31 KB/158010 for 89f73a4239a4d6e12de44ad55db0a5e8 in 2003ms, sequenceid=51, compaction requested=false 2024-12-09T06:15:44,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:44,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T06:15:44,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:44,413 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=154.31 KB heapSize=419.63 KB 2024-12-09T06:15:44,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:44,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/763b9257a77a41e69eefd63824843712 is 50, key is test_row_0/A:col10/1733724942789/Put/seqid=0 2024-12-09T06:15:44,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741843_1019 (size=12001) 2024-12-09T06:15:44,468 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/763b9257a77a41e69eefd63824843712 2024-12-09T06:15:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/753b5e3ce5de401ebc0bca82ae589cf6 is 50, key is test_row_0/B:col10/1733724942789/Put/seqid=0 2024-12-09T06:15:44,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741844_1020 (size=12001) 2024-12-09T06:15:44,538 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/753b5e3ce5de401ebc0bca82ae589cf6 2024-12-09T06:15:44,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/d373bf3d80b64868a7904049e24ee415 is 50, key is test_row_0/C:col10/1733724942789/Put/seqid=0 2024-12-09T06:15:44,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741845_1021 (size=12001) 2024-12-09T06:15:44,601 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/d373bf3d80b64868a7904049e24ee415 2024-12-09T06:15:44,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/763b9257a77a41e69eefd63824843712 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/763b9257a77a41e69eefd63824843712 2024-12-09T06:15:44,636 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/763b9257a77a41e69eefd63824843712, entries=150, sequenceid=120, filesize=11.7 K 2024-12-09T06:15:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/753b5e3ce5de401ebc0bca82ae589cf6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/753b5e3ce5de401ebc0bca82ae589cf6 2024-12-09T06:15:44,653 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/753b5e3ce5de401ebc0bca82ae589cf6, entries=150, sequenceid=120, filesize=11.7 K 2024-12-09T06:15:44,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/d373bf3d80b64868a7904049e24ee415 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/d373bf3d80b64868a7904049e24ee415 2024-12-09T06:15:44,678 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/d373bf3d80b64868a7904049e24ee415, entries=150, sequenceid=120, filesize=11.7 K 2024-12-09T06:15:44,680 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~419.58 KB/429648, currentSize=0 B/0 for 89f73a4239a4d6e12de44ad55db0a5e8 in 267ms, sequenceid=120, compaction requested=true 2024-12-09T06:15:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-09T06:15:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-09T06:15:44,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T06:15:44,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7140 sec 2024-12-09T06:15:44,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.7390 sec 2024-12-09T06:15:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:45,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:15:45,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:45,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:45,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:45,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:45,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:45,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:45,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/d5ed581db8cc425fbd8cf551fc2fe1e3 is 50, key is test_row_0/A:col10/1733724945002/Put/seqid=0 2024-12-09T06:15:45,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741846_1022 (size=12101) 2024-12-09T06:15:45,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733724945044, max timestamp=1733724945070Num uniques 859; 2024-12-09T06:15:45,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:15:45,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733724945044, max timestamp=1733724945070Num uniques 859; 2024-12-09T06:15:45,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:15:45,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733724945044, max timestamp=1733724945070Num uniques 859; 2024-12-09T06:15:45,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:15:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T06:15:45,138 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:45,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-09T06:15:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T06:15:45,142 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:45,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:45,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725005130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725005141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725005145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725005146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725005150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T06:15:45,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725005257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725005258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725005267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725005267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725005267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T06:15:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/d5ed581db8cc425fbd8cf551fc2fe1e3 2024-12-09T06:15:45,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T06:15:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T06:15:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3295a50f5f0b4a9ca0d070fbb1f84013 is 50, key is test_row_0/B:col10/1733724945002/Put/seqid=0 2024-12-09T06:15:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725005467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725005467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741847_1023 (size=12101) 2024-12-09T06:15:45,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3295a50f5f0b4a9ca0d070fbb1f84013 2024-12-09T06:15:45,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725005477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725005477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725005477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e13b6c4778ba4abf85f5f74d0229edbe is 50, key is test_row_0/C:col10/1733724945002/Put/seqid=0 2024-12-09T06:15:45,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741848_1024 (size=12101) 2024-12-09T06:15:45,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T06:15:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T06:15:45,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:45,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T06:15:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725005777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725005777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725005797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725005797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725005797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:45,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e13b6c4778ba4abf85f5f74d0229edbe 2024-12-09T06:15:45,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T06:15:45,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:45,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:45,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:45,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/d5ed581db8cc425fbd8cf551fc2fe1e3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d5ed581db8cc425fbd8cf551fc2fe1e3 2024-12-09T06:15:45,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d5ed581db8cc425fbd8cf551fc2fe1e3, entries=150, sequenceid=131, filesize=11.8 K 2024-12-09T06:15:45,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3295a50f5f0b4a9ca0d070fbb1f84013 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3295a50f5f0b4a9ca0d070fbb1f84013 2024-12-09T06:15:45,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3295a50f5f0b4a9ca0d070fbb1f84013, entries=150, sequenceid=131, filesize=11.8 K 2024-12-09T06:15:45,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e13b6c4778ba4abf85f5f74d0229edbe as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e13b6c4778ba4abf85f5f74d0229edbe 2024-12-09T06:15:45,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e13b6c4778ba4abf85f5f74d0229edbe, entries=150, sequenceid=131, filesize=11.8 K 2024-12-09T06:15:45,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 89f73a4239a4d6e12de44ad55db0a5e8 in 986ms, sequenceid=131, compaction requested=true 2024-12-09T06:15:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:46,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:15:46,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:46,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:46,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:46,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:46,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:46,014 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:15:46,014 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:15:46,022 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:15:46,024 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:15:46,024 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:46,024 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:46,025 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:46,025 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:46,026 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/7e82557b40104e538de666e2e04e6ad0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a8407467cb284a409d07c790479085b6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/753b5e3ce5de401ebc0bca82ae589cf6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3295a50f5f0b4a9ca0d070fbb1f84013] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=47.0 K 2024-12-09T06:15:46,027 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/aa44851b3fcb4567bded30e4c0754086, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0ea69aba66804923b80beb2b22c00221, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/763b9257a77a41e69eefd63824843712, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d5ed581db8cc425fbd8cf551fc2fe1e3] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=49.3 K 2024-12-09T06:15:46,028 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e82557b40104e538de666e2e04e6ad0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733724940999 2024-12-09T06:15:46,028 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa44851b3fcb4567bded30e4c0754086, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733724940999 2024-12-09T06:15:46,029 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a8407467cb284a409d07c790479085b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733724941441 2024-12-09T06:15:46,029 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ea69aba66804923b80beb2b22c00221, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733724941441 2024-12-09T06:15:46,030 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 753b5e3ce5de401ebc0bca82ae589cf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733724942757 2024-12-09T06:15:46,030 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763b9257a77a41e69eefd63824843712, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733724942757 2024-12-09T06:15:46,031 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3295a50f5f0b4a9ca0d070fbb1f84013, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733724944992 2024-12-09T06:15:46,031 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5ed581db8cc425fbd8cf551fc2fe1e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733724944992 2024-12-09T06:15:46,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T06:15:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:46,076 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T06:15:46,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:46,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:46,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:46,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:46,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:46,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:46,086 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:46,087 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:46,087 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/50ed96a0062d4778b7865827b9c8c8c8 is 50, key is test_row_0/B:col10/1733724945002/Put/seqid=0 2024-12-09T06:15:46,087 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/7629bb0f46f1494b921ffebdcc8944d0 is 50, key is test_row_0/A:col10/1733724945002/Put/seqid=0 2024-12-09T06:15:46,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/dd1a1fa1dc1b48938433a895d480610a is 50, key is test_row_0/A:col10/1733724945140/Put/seqid=0 2024-12-09T06:15:46,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741849_1025 (size=12239) 2024-12-09T06:15:46,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741850_1026 (size=12239) 2024-12-09T06:15:46,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741851_1027 (size=12151) 2024-12-09T06:15:46,160 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/dd1a1fa1dc1b48938433a895d480610a 2024-12-09T06:15:46,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/0923c64ed0d44a9f8b68a8b55f931096 is 50, key is test_row_0/B:col10/1733724945140/Put/seqid=0 2024-12-09T06:15:46,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741852_1028 (size=12151) 2024-12-09T06:15:46,204 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/0923c64ed0d44a9f8b68a8b55f931096 2024-12-09T06:15:46,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/0bb392ad45d84281921cefb21b785ad3 is 50, key is test_row_0/C:col10/1733724945140/Put/seqid=0 2024-12-09T06:15:46,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741853_1029 (size=12151) 2024-12-09T06:15:46,247 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/0bb392ad45d84281921cefb21b785ad3 2024-12-09T06:15:46,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/dd1a1fa1dc1b48938433a895d480610a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/dd1a1fa1dc1b48938433a895d480610a 2024-12-09T06:15:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T06:15:46,286 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/dd1a1fa1dc1b48938433a895d480610a, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T06:15:46,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/0923c64ed0d44a9f8b68a8b55f931096 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/0923c64ed0d44a9f8b68a8b55f931096 2024-12-09T06:15:46,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:46,309 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/0923c64ed0d44a9f8b68a8b55f931096, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T06:15:46,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/0bb392ad45d84281921cefb21b785ad3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/0bb392ad45d84281921cefb21b785ad3 2024-12-09T06:15:46,324 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/0bb392ad45d84281921cefb21b785ad3, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T06:15:46,330 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=40.25 KB/41220 for 89f73a4239a4d6e12de44ad55db0a5e8 in 254ms, sequenceid=168, compaction requested=true 2024-12-09T06:15:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:46,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T06:15:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:46,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:15:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-09T06:15:46,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:46,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:46,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:46,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:46,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:46,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:46,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T06:15:46,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1880 sec 2024-12-09T06:15:46,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1980 sec 2024-12-09T06:15:46,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/d33e03724a7a4b068aace3a54b120786 is 50, key is test_row_0/A:col10/1733724946308/Put/seqid=0 2024-12-09T06:15:46,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741854_1030 (size=12151) 2024-12-09T06:15:46,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/d33e03724a7a4b068aace3a54b120786 2024-12-09T06:15:46,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725006396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724946388, max timestamp=1733724946409Num uniques 859; 2024-12-09T06:15:46,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724946388, max timestamp=1733724946407Num uniques 859; 2024-12-09T06:15:46,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:15:46,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:15:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724946388, max timestamp=1733724946409Num uniques 859; 2024-12-09T06:15:46,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:15:46,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/bc5ae7bf1cc14626a0c6586269199902 is 50, key is test_row_0/B:col10/1733724946308/Put/seqid=0 2024-12-09T06:15:46,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725006450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725006451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741855_1031 (size=12151) 2024-12-09T06:15:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725006453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/bc5ae7bf1cc14626a0c6586269199902 2024-12-09T06:15:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725006459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e93bdd2e03134e6aae7e920f4b214fa9 is 50, key is test_row_0/C:col10/1733724946308/Put/seqid=0 2024-12-09T06:15:46,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741856_1032 (size=12151) 2024-12-09T06:15:46,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e93bdd2e03134e6aae7e920f4b214fa9 2024-12-09T06:15:46,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/d33e03724a7a4b068aace3a54b120786 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d33e03724a7a4b068aace3a54b120786 2024-12-09T06:15:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725006529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,536 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/50ed96a0062d4778b7865827b9c8c8c8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/50ed96a0062d4778b7865827b9c8c8c8 2024-12-09T06:15:46,537 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/7629bb0f46f1494b921ffebdcc8944d0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7629bb0f46f1494b921ffebdcc8944d0 2024-12-09T06:15:46,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d33e03724a7a4b068aace3a54b120786, entries=150, sequenceid=180, filesize=11.9 K 2024-12-09T06:15:46,559 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into 7629bb0f46f1494b921ffebdcc8944d0(size=12.0 K), total size for store is 35.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:46,559 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:46,559 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into 50ed96a0062d4778b7865827b9c8c8c8(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:46,559 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/bc5ae7bf1cc14626a0c6586269199902 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/bc5ae7bf1cc14626a0c6586269199902 2024-12-09T06:15:46,562 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=12, startTime=1733724946011; duration=0sec 2024-12-09T06:15:46,562 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:46,563 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:46,563 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T06:15:46,565 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=12, startTime=1733724945994; duration=0sec 2024-12-09T06:15:46,565 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:46,565 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:46,567 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60255 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T06:15:46,567 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:15:46,567 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:46,568 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/92bb28539f8a43ce9f87976254bda142, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/752035daae394490808b33b382c82d10, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/d373bf3d80b64868a7904049e24ee415, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e13b6c4778ba4abf85f5f74d0229edbe, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/0bb392ad45d84281921cefb21b785ad3] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=58.8 K 2024-12-09T06:15:46,569 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bb28539f8a43ce9f87976254bda142, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733724940999 2024-12-09T06:15:46,570 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 752035daae394490808b33b382c82d10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733724941441 2024-12-09T06:15:46,571 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting d373bf3d80b64868a7904049e24ee415, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733724942757 2024-12-09T06:15:46,572 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e13b6c4778ba4abf85f5f74d0229edbe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733724944992 2024-12-09T06:15:46,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725006567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,573 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb392ad45d84281921cefb21b785ad3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733724945098 2024-12-09T06:15:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725006567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725006567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/bc5ae7bf1cc14626a0c6586269199902, entries=150, sequenceid=180, filesize=11.9 K 2024-12-09T06:15:46,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e93bdd2e03134e6aae7e920f4b214fa9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e93bdd2e03134e6aae7e920f4b214fa9 2024-12-09T06:15:46,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725006577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e93bdd2e03134e6aae7e920f4b214fa9, entries=150, sequenceid=180, filesize=11.9 K 2024-12-09T06:15:46,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=119.58 KB/122445 for 89f73a4239a4d6e12de44ad55db0a5e8 in 263ms, sequenceid=180, compaction requested=true 2024-12-09T06:15:46,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:46,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:46,595 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:46,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:46,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:46,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:46,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:46,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:15:46,597 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:46,597 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:46,597 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:46,598 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7629bb0f46f1494b921ffebdcc8944d0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/dd1a1fa1dc1b48938433a895d480610a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d33e03724a7a4b068aace3a54b120786] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=35.7 K 2024-12-09T06:15:46,598 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7629bb0f46f1494b921ffebdcc8944d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733724944992 2024-12-09T06:15:46,599 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd1a1fa1dc1b48938433a895d480610a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733724945098 2024-12-09T06:15:46,600 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting d33e03724a7a4b068aace3a54b120786, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733724946308 2024-12-09T06:15:46,604 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:46,605 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/daff047328fc4beaaedf299ea9cfd6dd is 50, key is test_row_0/C:col10/1733724945140/Put/seqid=0 2024-12-09T06:15:46,619 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:46,619 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/b29dc5dd204147f693fd553e7eebd79b is 50, key is test_row_0/A:col10/1733724946308/Put/seqid=0 2024-12-09T06:15:46,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741857_1033 (size=12323) 2024-12-09T06:15:46,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741858_1034 (size=12391) 2024-12-09T06:15:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:46,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=126.28 KB heapSize=362.32 KB 2024-12-09T06:15:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:46,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/15a8a5f764e947f8adf30e5e18d9e287 is 50, key is test_row_0/A:col10/1733724946447/Put/seqid=0 2024-12-09T06:15:46,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741859_1035 (size=14541) 2024-12-09T06:15:46,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725006782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725006782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725006787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725006790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725006797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725006897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725006897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:46,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725006907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:46,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,033 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/daff047328fc4beaaedf299ea9cfd6dd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/daff047328fc4beaaedf299ea9cfd6dd 2024-12-09T06:15:47,046 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into daff047328fc4beaaedf299ea9cfd6dd(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:47,046 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:47,046 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=11, startTime=1733724946012; duration=0sec 2024-12-09T06:15:47,047 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:15:47,047 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:47,047 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:47,047 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:47,048 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:47,048 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:47,049 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,049 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/50ed96a0062d4778b7865827b9c8c8c8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/0923c64ed0d44a9f8b68a8b55f931096, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/bc5ae7bf1cc14626a0c6586269199902] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=35.7 K 2024-12-09T06:15:47,050 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ed96a0062d4778b7865827b9c8c8c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733724944992 2024-12-09T06:15:47,051 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/b29dc5dd204147f693fd553e7eebd79b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b29dc5dd204147f693fd553e7eebd79b 2024-12-09T06:15:47,051 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 0923c64ed0d44a9f8b68a8b55f931096, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733724945098 2024-12-09T06:15:47,053 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5ae7bf1cc14626a0c6586269199902, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733724946308 2024-12-09T06:15:47,070 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into b29dc5dd204147f693fd553e7eebd79b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:47,070 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:47,070 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=13, startTime=1733724946594; duration=0sec 2024-12-09T06:15:47,071 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:47,071 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:47,073 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:15:47,075 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:15:47,076 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:15:47,076 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. because compaction request was cancelled 2024-12-09T06:15:47,076 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:47,088 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#23 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:47,089 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/b03ce88d8e204a50b26f3cbfc1cc6853 is 50, key is test_row_0/B:col10/1733724946308/Put/seqid=0 2024-12-09T06:15:47,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741860_1036 (size=12391) 2024-12-09T06:15:47,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725007097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,112 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/b03ce88d8e204a50b26f3cbfc1cc6853 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b03ce88d8e204a50b26f3cbfc1cc6853 2024-12-09T06:15:47,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725007107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725007107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725007109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,129 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into b03ce88d8e204a50b26f3cbfc1cc6853(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:47,129 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:47,129 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=13, startTime=1733724946595; duration=0sec 2024-12-09T06:15:47,130 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:47,130 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:47,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725007127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/15a8a5f764e947f8adf30e5e18d9e287 2024-12-09T06:15:47,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a7669c9c013b4692ad388de28b2987c1 is 50, key is test_row_0/B:col10/1733724946447/Put/seqid=0 2024-12-09T06:15:47,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741861_1037 (size=12151) 2024-12-09T06:15:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T06:15:47,287 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:47,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-09T06:15:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T06:15:47,291 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:47,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:47,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T06:15:47,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725007427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725007427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725007436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T06:15:47,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:47,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T06:15:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:47,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T06:15:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725007607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a7669c9c013b4692ad388de28b2987c1 2024-12-09T06:15:47,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725007626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/beacae8e0d4a48d7b3dc4d8459c18b99 is 50, key is test_row_0/C:col10/1733724946447/Put/seqid=0 2024-12-09T06:15:47,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741862_1038 (size=12151) 2024-12-09T06:15:47,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T06:15:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T06:15:47,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:47,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:47,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T06:15:47,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725007937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725007937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:47,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725007946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:48,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/beacae8e0d4a48d7b3dc4d8459c18b99 2024-12-09T06:15:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/15a8a5f764e947f8adf30e5e18d9e287 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/15a8a5f764e947f8adf30e5e18d9e287 2024-12-09T06:15:48,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T06:15:48,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/15a8a5f764e947f8adf30e5e18d9e287, entries=200, sequenceid=218, filesize=14.2 K 2024-12-09T06:15:48,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:48,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:48,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:48,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:48,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:48,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a7669c9c013b4692ad388de28b2987c1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a7669c9c013b4692ad388de28b2987c1 2024-12-09T06:15:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:48,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a7669c9c013b4692ad388de28b2987c1, entries=150, sequenceid=218, filesize=11.9 K 2024-12-09T06:15:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/beacae8e0d4a48d7b3dc4d8459c18b99 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/beacae8e0d4a48d7b3dc4d8459c18b99 2024-12-09T06:15:48,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/beacae8e0d4a48d7b3dc4d8459c18b99, entries=150, sequenceid=218, filesize=11.9 K 2024-12-09T06:15:48,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~126.28 KB/129315, heapSize ~362.27 KB/370968, currentSize=53.67 KB/54960 for 89f73a4239a4d6e12de44ad55db0a5e8 in 1342ms, sequenceid=218, compaction requested=true 2024-12-09T06:15:48,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:48,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:15:48,095 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:48,095 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. because compaction request was cancelled 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. because compaction request was cancelled 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:48,096 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:48,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:48,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:48,097 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36625 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:48,097 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:15:48,097 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:48,098 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/daff047328fc4beaaedf299ea9cfd6dd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e93bdd2e03134e6aae7e920f4b214fa9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/beacae8e0d4a48d7b3dc4d8459c18b99] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=35.8 K 2024-12-09T06:15:48,098 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting daff047328fc4beaaedf299ea9cfd6dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733724945098 2024-12-09T06:15:48,101 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e93bdd2e03134e6aae7e920f4b214fa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733724946308 2024-12-09T06:15:48,102 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting beacae8e0d4a48d7b3dc4d8459c18b99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733724946447 2024-12-09T06:15:48,115 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:48,116 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/1ae2315e8206434abdcaeb158762d500 is 50, key is test_row_0/C:col10/1733724946447/Put/seqid=0 2024-12-09T06:15:48,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741863_1039 (size=12425) 2024-12-09T06:15:48,138 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/1ae2315e8206434abdcaeb158762d500 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/1ae2315e8206434abdcaeb158762d500 2024-12-09T06:15:48,149 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into 1ae2315e8206434abdcaeb158762d500(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:48,149 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:48,149 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=13, startTime=1733724948095; duration=0sec 2024-12-09T06:15:48,149 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:48,149 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:48,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T06:15:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:48,224 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:15:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:48,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/af39531140a541e697ddd65f9fc8eb67 is 50, key is test_row_0/A:col10/1733724946779/Put/seqid=0 2024-12-09T06:15:48,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741864_1040 (size=12151) 2024-12-09T06:15:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T06:15:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:48,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:48,638 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/af39531140a541e697ddd65f9fc8eb67 2024-12-09T06:15:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/2873d493714a458e9862814b16300fba is 50, key is test_row_0/B:col10/1733724946779/Put/seqid=0 2024-12-09T06:15:48,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724948649, max timestamp=1733724948664Num uniques 859; 2024-12-09T06:15:48,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724948649, max timestamp=1733724948664Num uniques 859; 2024-12-09T06:15:48,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:15:48,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:15:48,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724948649, max timestamp=1733724948664Num uniques 859; 2024-12-09T06:15:48,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:15:48,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741865_1041 (size=12151) 2024-12-09T06:15:48,678 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/2873d493714a458e9862814b16300fba 2024-12-09T06:15:48,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/2d7b4a8fbaac425b88b999430ea88da1 is 50, key is test_row_0/C:col10/1733724946779/Put/seqid=0 2024-12-09T06:15:48,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741866_1042 (size=12151) 2024-12-09T06:15:48,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725008722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:48,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725008724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:48,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725008836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725008837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725008947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725008947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:48,952 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:48,952 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:48,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725008966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:48,968 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725009047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725009047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:49,098 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/2d7b4a8fbaac425b88b999430ea88da1 2024-12-09T06:15:49,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/af39531140a541e697ddd65f9fc8eb67 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/af39531140a541e697ddd65f9fc8eb67 2024-12-09T06:15:49,112 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/af39531140a541e697ddd65f9fc8eb67, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T06:15:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/2873d493714a458e9862814b16300fba as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/2873d493714a458e9862814b16300fba 2024-12-09T06:15:49,124 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/2873d493714a458e9862814b16300fba, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T06:15:49,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/2d7b4a8fbaac425b88b999430ea88da1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/2d7b4a8fbaac425b88b999430ea88da1 2024-12-09T06:15:49,135 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/2d7b4a8fbaac425b88b999430ea88da1, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T06:15:49,136 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 89f73a4239a4d6e12de44ad55db0a5e8 in 912ms, sequenceid=233, compaction requested=true 2024-12-09T06:15:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T06:15:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-09T06:15:49,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T06:15:49,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8450 sec 2024-12-09T06:15:49,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.8560 sec 2024-12-09T06:15:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:49,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T06:15:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:49,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:49,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:49,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:49,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:49,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:49,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/0d794cb8a6164141a9c75b7330db2ad0 is 50, key is test_row_0/A:col10/1733724949360/Put/seqid=0 2024-12-09T06:15:49,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:49,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725009378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:49,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725009381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:49,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741867_1043 (size=14741) 2024-12-09T06:15:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:49,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/0d794cb8a6164141a9c75b7330db2ad0 2024-12-09T06:15:49,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/f0479acd98f849009d2597c1be224efb is 50, key is test_row_0/B:col10/1733724949360/Put/seqid=0 2024-12-09T06:15:49,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741868_1044 (size=12301) 2024-12-09T06:15:49,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/f0479acd98f849009d2597c1be224efb 2024-12-09T06:15:49,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/b9d157b16a0f4c4d95ab7bc678d3ac11 is 50, key is test_row_0/C:col10/1733724949360/Put/seqid=0 2024-12-09T06:15:49,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741869_1045 (size=12301) 2024-12-09T06:15:49,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/b9d157b16a0f4c4d95ab7bc678d3ac11 2024-12-09T06:15:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T06:15:49,437 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:49,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-09T06:15:49,443 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T06:15:49,445 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:49,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:49,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/0d794cb8a6164141a9c75b7330db2ad0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0d794cb8a6164141a9c75b7330db2ad0 2024-12-09T06:15:49,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0d794cb8a6164141a9c75b7330db2ad0, entries=200, sequenceid=270, filesize=14.4 K 2024-12-09T06:15:49,471 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-09T06:15:49,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/f0479acd98f849009d2597c1be224efb as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/f0479acd98f849009d2597c1be224efb 2024-12-09T06:15:49,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/f0479acd98f849009d2597c1be224efb, entries=150, sequenceid=270, filesize=12.0 K 2024-12-09T06:15:49,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/b9d157b16a0f4c4d95ab7bc678d3ac11 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b9d157b16a0f4c4d95ab7bc678d3ac11 2024-12-09T06:15:49,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725009487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:49,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725009487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:49,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:49,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:49,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b9d157b16a0f4c4d95ab7bc678d3ac11, entries=150, sequenceid=270, filesize=12.0 K 2024-12-09T06:15:49,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 89f73a4239a4d6e12de44ad55db0a5e8 in 147ms, sequenceid=270, compaction requested=true 2024-12-09T06:15:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:15:49,508 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:15:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:49,508 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:15:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:49,510 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53824 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:15:49,511 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:49,511 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:49,511 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b29dc5dd204147f693fd553e7eebd79b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/15a8a5f764e947f8adf30e5e18d9e287, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/af39531140a541e697ddd65f9fc8eb67, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0d794cb8a6164141a9c75b7330db2ad0] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=52.6 K 2024-12-09T06:15:49,512 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48994 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:15:49,512 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:49,512 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:49,512 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b03ce88d8e204a50b26f3cbfc1cc6853, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a7669c9c013b4692ad388de28b2987c1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/2873d493714a458e9862814b16300fba, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/f0479acd98f849009d2597c1be224efb] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=47.8 K 2024-12-09T06:15:49,513 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting b29dc5dd204147f693fd553e7eebd79b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733724946308 2024-12-09T06:15:49,515 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b03ce88d8e204a50b26f3cbfc1cc6853, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733724946308 2024-12-09T06:15:49,515 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a8a5f764e947f8adf30e5e18d9e287, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733724946440 2024-12-09T06:15:49,516 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a7669c9c013b4692ad388de28b2987c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733724946447 2024-12-09T06:15:49,516 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting af39531140a541e697ddd65f9fc8eb67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733724946767 2024-12-09T06:15:49,517 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d794cb8a6164141a9c75b7330db2ad0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733724948713 2024-12-09T06:15:49,517 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2873d493714a458e9862814b16300fba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733724946767 2024-12-09T06:15:49,518 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f0479acd98f849009d2597c1be224efb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733724948722 2024-12-09T06:15:49,555 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:49,556 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/ffdb9d22cf5f4936b4262df26988809a is 50, key is test_row_0/B:col10/1733724949360/Put/seqid=0 2024-12-09T06:15:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T06:15:49,559 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:49,560 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/41dba62976cf4a13b3ad58644cf3cf93 is 50, key is test_row_0/A:col10/1733724949360/Put/seqid=0 2024-12-09T06:15:49,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741870_1046 (size=12677) 2024-12-09T06:15:49,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741871_1047 (size=12677) 2024-12-09T06:15:49,593 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/ffdb9d22cf5f4936b4262df26988809a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ffdb9d22cf5f4936b4262df26988809a 2024-12-09T06:15:49,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T06:15:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:49,600 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T06:15:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:49,600 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/41dba62976cf4a13b3ad58644cf3cf93 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/41dba62976cf4a13b3ad58644cf3cf93 2024-12-09T06:15:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/167dcf53775c471a8c6aeb2777a09915 is 50, key is test_row_1/A:col10/1733724949375/Put/seqid=0 2024-12-09T06:15:49,610 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into ffdb9d22cf5f4936b4262df26988809a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:49,610 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:49,611 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=12, startTime=1733724949508; duration=0sec 2024-12-09T06:15:49,611 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into 41dba62976cf4a13b3ad58644cf3cf93(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:49,611 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:49,612 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=12, startTime=1733724949508; duration=0sec 2024-12-09T06:15:49,612 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:49,612 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:49,611 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:49,612 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:49,612 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:49,615 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:49,615 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:15:49,615 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:49,615 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/1ae2315e8206434abdcaeb158762d500, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/2d7b4a8fbaac425b88b999430ea88da1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b9d157b16a0f4c4d95ab7bc678d3ac11] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=36.0 K 2024-12-09T06:15:49,616 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae2315e8206434abdcaeb158762d500, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733724946447 2024-12-09T06:15:49,617 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d7b4a8fbaac425b88b999430ea88da1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733724946767 2024-12-09T06:15:49,617 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9d157b16a0f4c4d95ab7bc678d3ac11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733724948722 2024-12-09T06:15:49,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741872_1048 (size=9857) 2024-12-09T06:15:49,630 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#36 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:49,631 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/744b509a8d8940b08afaecfc27d9962a is 50, key is test_row_0/C:col10/1733724949360/Put/seqid=0 2024-12-09T06:15:49,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741873_1049 (size=12677) 2024-12-09T06:15:49,658 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/744b509a8d8940b08afaecfc27d9962a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/744b509a8d8940b08afaecfc27d9962a 2024-12-09T06:15:49,670 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into 744b509a8d8940b08afaecfc27d9962a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:49,670 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:49,670 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=13, startTime=1733724949508; duration=0sec 2024-12-09T06:15:49,670 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:49,670 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:49,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:49,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733724949728, max timestamp=1733724949739Num uniques 859; 2024-12-09T06:15:49,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733724949728, max timestamp=1733724949739Num uniques 859; 2024-12-09T06:15:49,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:15:49,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:15:49,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733724949728, max timestamp=1733724949739Num uniques 859; 2024-12-09T06:15:49,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:15:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T06:15:49,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724949776, max timestamp=1733724949783Num uniques 1018; 2024-12-09T06:15:49,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:15:49,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724949776, max timestamp=1733724949783Num uniques 1018; 2024-12-09T06:15:49,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:15:49,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724949776, max timestamp=1733724949783Num uniques 1018; 2024-12-09T06:15:49,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:15:49,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733724949807, max timestamp=1733724949830Num uniques 1027; 2024-12-09T06:15:49,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733724949807, max timestamp=1733724949830Num uniques 1027; 2024-12-09T06:15:49,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733724949807, max timestamp=1733724949830Num uniques 1027; 2024-12-09T06:15:49,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T06:15:49,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T06:15:49,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T06:15:49,881 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T06:15:49,881 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T06:15:49,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733724949864, max timestamp=1733724949883Num uniques 1086; 2024-12-09T06:15:49,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733724949864, max timestamp=1733724949883Num uniques 1086; 2024-12-09T06:15:49,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T06:15:49,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T06:15:49,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733724949864, max timestamp=1733724949883Num uniques 1086; 2024-12-09T06:15:49,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T06:15:49,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725009936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725009937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,031 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/167dcf53775c471a8c6aeb2777a09915 2024-12-09T06:15:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/509ed62ca73a438182574a53cdcd51c3 is 50, key is test_row_1/B:col10/1733724949375/Put/seqid=0 2024-12-09T06:15:50,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725010047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741874_1050 (size=9857) 2024-12-09T06:15:50,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725010047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,052 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/509ed62ca73a438182574a53cdcd51c3 2024-12-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/5d3afd0c182844778518e5f4fbc2d09a is 50, key is test_row_1/C:col10/1733724949375/Put/seqid=0 2024-12-09T06:15:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T06:15:50,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741875_1051 (size=9857) 2024-12-09T06:15:50,089 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/5d3afd0c182844778518e5f4fbc2d09a 2024-12-09T06:15:50,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/167dcf53775c471a8c6aeb2777a09915 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/167dcf53775c471a8c6aeb2777a09915 2024-12-09T06:15:50,113 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/167dcf53775c471a8c6aeb2777a09915, entries=100, sequenceid=279, filesize=9.6 K 2024-12-09T06:15:50,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/509ed62ca73a438182574a53cdcd51c3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/509ed62ca73a438182574a53cdcd51c3 2024-12-09T06:15:50,126 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/509ed62ca73a438182574a53cdcd51c3, entries=100, sequenceid=279, filesize=9.6 K 2024-12-09T06:15:50,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/5d3afd0c182844778518e5f4fbc2d09a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/5d3afd0c182844778518e5f4fbc2d09a 2024-12-09T06:15:50,142 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/5d3afd0c182844778518e5f4fbc2d09a, entries=100, sequenceid=279, filesize=9.6 K 2024-12-09T06:15:50,144 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 89f73a4239a4d6e12de44ad55db0a5e8 in 544ms, sequenceid=279, compaction requested=false 2024-12-09T06:15:50,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:50,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:50,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T06:15:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-09T06:15:50,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T06:15:50,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 702 msec 2024-12-09T06:15:50,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 712 msec 2024-12-09T06:15:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:50,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=147.60 KB heapSize=418.31 KB 2024-12-09T06:15:50,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:50,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:15:50,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:50,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:15:50,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:50,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:15:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725010269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725010271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/bfa8952c97ca4860afc14f58ebd2a6d7 is 50, key is test_row_0/A:col10/1733724949933/Put/seqid=0 2024-12-09T06:15:50,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741876_1052 (size=14741) 2024-12-09T06:15:50,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/bfa8952c97ca4860afc14f58ebd2a6d7 2024-12-09T06:15:50,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/68bf1ac7f731425ca52ff17a06b0477e is 50, key is test_row_0/B:col10/1733724949933/Put/seqid=0 2024-12-09T06:15:50,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741877_1053 (size=12301) 2024-12-09T06:15:50,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/68bf1ac7f731425ca52ff17a06b0477e 2024-12-09T06:15:50,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/9ec976c4961146ae897d767ad0d9c05a is 50, key is test_row_0/C:col10/1733724949933/Put/seqid=0 2024-12-09T06:15:50,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741878_1054 (size=12301) 2024-12-09T06:15:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725010377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725010377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T06:15:50,587 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:50,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-09T06:15:50,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,589 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725010587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T06:15:50,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725010587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T06:15:50,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/9ec976c4961146ae897d767ad0d9c05a 2024-12-09T06:15:50,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/bfa8952c97ca4860afc14f58ebd2a6d7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bfa8952c97ca4860afc14f58ebd2a6d7 2024-12-09T06:15:50,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T06:15:50,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:50,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:50,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:50,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:50,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:50,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bfa8952c97ca4860afc14f58ebd2a6d7, entries=200, sequenceid=371, filesize=14.4 K 2024-12-09T06:15:50,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/68bf1ac7f731425ca52ff17a06b0477e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/68bf1ac7f731425ca52ff17a06b0477e 2024-12-09T06:15:50,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/68bf1ac7f731425ca52ff17a06b0477e, entries=150, sequenceid=371, filesize=12.0 K 2024-12-09T06:15:50,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/9ec976c4961146ae897d767ad0d9c05a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/9ec976c4961146ae897d767ad0d9c05a 2024-12-09T06:15:50,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/9ec976c4961146ae897d767ad0d9c05a, entries=150, sequenceid=371, filesize=12.0 K 2024-12-09T06:15:50,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~418.27 KB/428304, currentSize=33.54 KB/34350 for 89f73a4239a4d6e12de44ad55db0a5e8 in 508ms, sequenceid=371, compaction requested=true 2024-12-09T06:15:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:15:50,768 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:50,768 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:50,770 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:50,770 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:50,770 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:50,770 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ffdb9d22cf5f4936b4262df26988809a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/509ed62ca73a438182574a53cdcd51c3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/68bf1ac7f731425ca52ff17a06b0477e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=34.0 K 2024-12-09T06:15:50,770 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37275 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:50,770 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:50,770 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:50,771 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/41dba62976cf4a13b3ad58644cf3cf93, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/167dcf53775c471a8c6aeb2777a09915, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bfa8952c97ca4860afc14f58ebd2a6d7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=36.4 K 2024-12-09T06:15:50,771 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdb9d22cf5f4936b4262df26988809a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733724948722 2024-12-09T06:15:50,771 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41dba62976cf4a13b3ad58644cf3cf93, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733724948722 2024-12-09T06:15:50,772 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 509ed62ca73a438182574a53cdcd51c3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733724949375 2024-12-09T06:15:50,772 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 167dcf53775c471a8c6aeb2777a09915, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733724949375 2024-12-09T06:15:50,772 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 68bf1ac7f731425ca52ff17a06b0477e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733724949923 2024-12-09T06:15:50,772 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa8952c97ca4860afc14f58ebd2a6d7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733724949923 2024-12-09T06:15:50,807 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:50,808 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/b50f4212464348b097932310e027315b is 50, key is test_row_0/A:col10/1733724949933/Put/seqid=0 2024-12-09T06:15:50,815 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:50,816 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/cb8fc737db2845989010eb56cd9c132c is 50, key is test_row_0/B:col10/1733724949933/Put/seqid=0 2024-12-09T06:15:50,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741879_1055 (size=12779) 2024-12-09T06:15:50,835 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/b50f4212464348b097932310e027315b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b50f4212464348b097932310e027315b 2024-12-09T06:15:50,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741880_1056 (size=12779) 2024-12-09T06:15:50,848 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into b50f4212464348b097932310e027315b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:50,849 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:50,849 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=13, startTime=1733724950768; duration=0sec 2024-12-09T06:15:50,849 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:50,849 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:50,849 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:50,851 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:50,851 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:15:50,851 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:50,852 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/744b509a8d8940b08afaecfc27d9962a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/5d3afd0c182844778518e5f4fbc2d09a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/9ec976c4961146ae897d767ad0d9c05a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=34.0 K 2024-12-09T06:15:50,852 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744b509a8d8940b08afaecfc27d9962a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733724948722 2024-12-09T06:15:50,853 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3afd0c182844778518e5f4fbc2d09a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733724949375 2024-12-09T06:15:50,854 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec976c4961146ae897d767ad0d9c05a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733724949923 2024-12-09T06:15:50,867 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#44 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:50,867 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/4182bdfabe13404db6446c95a91fb21b is 50, key is test_row_0/C:col10/1733724949933/Put/seqid=0 2024-12-09T06:15:50,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741881_1057 (size=12779) 2024-12-09T06:15:50,882 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/4182bdfabe13404db6446c95a91fb21b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/4182bdfabe13404db6446c95a91fb21b 2024-12-09T06:15:50,895 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into 4182bdfabe13404db6446c95a91fb21b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:50,895 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:50,895 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=13, startTime=1733724950768; duration=0sec 2024-12-09T06:15:50,896 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:50,896 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:50,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T06:15:50,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:50,899 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T06:15:50,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:50,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:50,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:50,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:50,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:50,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:50,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T06:15:50,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/5218bbd299814ab2bf930582eb279d68 is 50, key is test_row_0/A:col10/1733724950897/Put/seqid=0 2024-12-09T06:15:50,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741882_1058 (size=12297) 2024-12-09T06:15:50,920 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/5218bbd299814ab2bf930582eb279d68 2024-12-09T06:15:50,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/071e2c67978e4d33ae5a4d036c1f5684 is 50, key is test_row_0/B:col10/1733724950897/Put/seqid=0 2024-12-09T06:15:50,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724950917, max timestamp=1733724950941Num uniques 859; 2024-12-09T06:15:50,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724950917, max timestamp=1733724950941Num uniques 859; 2024-12-09T06:15:50,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:15:50,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:15:50,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741883_1059 (size=9857) 2024-12-09T06:15:50,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725010972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725010976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725010976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,981 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:50,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725010977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,982 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725010987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:50,990 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725011087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:51,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725011087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T06:15:51,248 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/cb8fc737db2845989010eb56cd9c132c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/cb8fc737db2845989010eb56cd9c132c 2024-12-09T06:15:51,255 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into cb8fc737db2845989010eb56cd9c132c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:51,256 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:51,256 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=13, startTime=1733724950768; duration=0sec 2024-12-09T06:15:51,256 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:51,256 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725011297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725011297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,381 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/071e2c67978e4d33ae5a4d036c1f5684 2024-12-09T06:15:51,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/bc828cc6f6f340abb6cb7e805597b4aa is 50, key is test_row_0/C:col10/1733724950897/Put/seqid=0 2024-12-09T06:15:51,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741884_1060 (size=9857) 2024-12-09T06:15:51,401 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/bc828cc6f6f340abb6cb7e805597b4aa 2024-12-09T06:15:51,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/5218bbd299814ab2bf930582eb279d68 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5218bbd299814ab2bf930582eb279d68 2024-12-09T06:15:51,416 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5218bbd299814ab2bf930582eb279d68, entries=150, sequenceid=382, filesize=12.0 K 2024-12-09T06:15:51,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/071e2c67978e4d33ae5a4d036c1f5684 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/071e2c67978e4d33ae5a4d036c1f5684 2024-12-09T06:15:51,426 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/071e2c67978e4d33ae5a4d036c1f5684, entries=100, sequenceid=382, filesize=9.6 K 2024-12-09T06:15:51,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/bc828cc6f6f340abb6cb7e805597b4aa as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/bc828cc6f6f340abb6cb7e805597b4aa 2024-12-09T06:15:51,445 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/bc828cc6f6f340abb6cb7e805597b4aa, entries=100, sequenceid=382, filesize=9.6 K 2024-12-09T06:15:51,447 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 89f73a4239a4d6e12de44ad55db0a5e8 in 548ms, sequenceid=382, compaction requested=false 2024-12-09T06:15:51,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:51,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:51,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T06:15:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-09T06:15:51,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T06:15:51,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 859 msec 2024-12-09T06:15:51,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 867 msec 2024-12-09T06:15:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:51,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=147.60 KB heapSize=423.55 KB 2024-12-09T06:15:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:51,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:51,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:51,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:51,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/f14f55df06bd4faf908a959c3fce6e69 is 50, key is test_row_0/A:col10/1733724951608/Put/seqid=0 2024-12-09T06:15:51,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741885_1061 (size=14741) 2024-12-09T06:15:51,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725011626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/f14f55df06bd4faf908a959c3fce6e69 2024-12-09T06:15:51,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725011627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/5740c30009dc41c9ad38a643660f50f8 is 50, key is test_row_0/B:col10/1733724951608/Put/seqid=0 2024-12-09T06:15:51,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741886_1062 (size=12301) 2024-12-09T06:15:51,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/5740c30009dc41c9ad38a643660f50f8 2024-12-09T06:15:51,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/86da9095f4144624ac304ea1eef8919e is 50, key is test_row_0/C:col10/1733724951608/Put/seqid=0 2024-12-09T06:15:51,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741887_1063 (size=12301) 2024-12-09T06:15:51,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/86da9095f4144624ac304ea1eef8919e 2024-12-09T06:15:51,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/f14f55df06bd4faf908a959c3fce6e69 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f14f55df06bd4faf908a959c3fce6e69 2024-12-09T06:15:51,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f14f55df06bd4faf908a959c3fce6e69, entries=200, sequenceid=417, filesize=14.4 K 2024-12-09T06:15:51,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/5740c30009dc41c9ad38a643660f50f8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5740c30009dc41c9ad38a643660f50f8 2024-12-09T06:15:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T06:15:51,727 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:51,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-09T06:15:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T06:15:51,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:51,736 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:51,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:51,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725011737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:51,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5740c30009dc41c9ad38a643660f50f8, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T06:15:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/86da9095f4144624ac304ea1eef8919e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/86da9095f4144624ac304ea1eef8919e 2024-12-09T06:15:51,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725011746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:51,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/86da9095f4144624ac304ea1eef8919e, entries=150, sequenceid=417, filesize=12.0 K 2024-12-09T06:15:51,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~443.42 KB/454064, currentSize=33.54 KB/34350 for 89f73a4239a4d6e12de44ad55db0a5e8 in 145ms, sequenceid=417, compaction requested=true 2024-12-09T06:15:51,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:51,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:15:51,754 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:51,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:51,754 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:51,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:51,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:51,756 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:51,756 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39817 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:51,756 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:51,756 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:51,756 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:51,756 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:51,756 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/cb8fc737db2845989010eb56cd9c132c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/071e2c67978e4d33ae5a4d036c1f5684, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5740c30009dc41c9ad38a643660f50f8] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=34.1 K 2024-12-09T06:15:51,756 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b50f4212464348b097932310e027315b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5218bbd299814ab2bf930582eb279d68, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f14f55df06bd4faf908a959c3fce6e69] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=38.9 K 2024-12-09T06:15:51,757 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting b50f4212464348b097932310e027315b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733724949923 2024-12-09T06:15:51,757 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting cb8fc737db2845989010eb56cd9c132c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733724949923 2024-12-09T06:15:51,758 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 071e2c67978e4d33ae5a4d036c1f5684, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733724950269 2024-12-09T06:15:51,758 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5218bbd299814ab2bf930582eb279d68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733724950267 2024-12-09T06:15:51,758 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5740c30009dc41c9ad38a643660f50f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733724950965 2024-12-09T06:15:51,758 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting f14f55df06bd4faf908a959c3fce6e69, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733724950965 2024-12-09T06:15:51,766 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:51,767 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:51,768 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a221b4774269407aab459a383c63c35b is 50, key is test_row_0/B:col10/1733724951608/Put/seqid=0 2024-12-09T06:15:51,768 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/2dedcf085d3e424ba7734ecba2dd9671 is 50, key is test_row_0/A:col10/1733724951608/Put/seqid=0 2024-12-09T06:15:51,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741889_1065 (size=12881) 2024-12-09T06:15:51,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741888_1064 (size=12881) 2024-12-09T06:15:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T06:15:51,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T06:15:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:51,892 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T06:15:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/47fd9923c33a4cf38404324ed79f7c1a is 50, key is test_row_0/A:col10/1733724951620/Put/seqid=0 2024-12-09T06:15:51,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741890_1066 (size=9857) 2024-12-09T06:15:51,904 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/47fd9923c33a4cf38404324ed79f7c1a 2024-12-09T06:15:51,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/64dc10953972409b87715b63ae6fb9f3 is 50, key is test_row_0/B:col10/1733724951620/Put/seqid=0 2024-12-09T06:15:51,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741891_1067 (size=9857) 2024-12-09T06:15:51,937 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/64dc10953972409b87715b63ae6fb9f3 2024-12-09T06:15:51,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/473e5e617d6e419d849544c945942e27 is 50, key is test_row_0/C:col10/1733724951620/Put/seqid=0 2024-12-09T06:15:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:51,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741892_1068 (size=9857) 2024-12-09T06:15:51,967 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/473e5e617d6e419d849544c945942e27 2024-12-09T06:15:51,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/47fd9923c33a4cf38404324ed79f7c1a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/47fd9923c33a4cf38404324ed79f7c1a 2024-12-09T06:15:51,989 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/47fd9923c33a4cf38404324ed79f7c1a, entries=100, sequenceid=425, filesize=9.6 K 2024-12-09T06:15:51,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/64dc10953972409b87715b63ae6fb9f3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/64dc10953972409b87715b63ae6fb9f3 2024-12-09T06:15:52,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733724951975, max timestamp=1733724952000Num uniques 859; 2024-12-09T06:15:52,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733724951975, max timestamp=1733724952000Num uniques 859; 2024-12-09T06:15:52,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733724951975, max timestamp=1733724952000Num uniques 859; 2024-12-09T06:15:52,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T06:15:52,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T06:15:52,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T06:15:52,006 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/64dc10953972409b87715b63ae6fb9f3, entries=100, sequenceid=425, filesize=9.6 K 2024-12-09T06:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/473e5e617d6e419d849544c945942e27 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/473e5e617d6e419d849544c945942e27 2024-12-09T06:15:52,017 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/473e5e617d6e419d849544c945942e27, entries=100, sequenceid=425, filesize=9.6 K 2024-12-09T06:15:52,018 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=73.80 KB/75570 for 89f73a4239a4d6e12de44ad55db0a5e8 in 127ms, sequenceid=425, compaction requested=true 2024-12-09T06:15:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T06:15:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-09T06:15:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:52,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=87.22 KB heapSize=231.47 KB 2024-12-09T06:15:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:15:52,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T06:15:52,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 284 msec 2024-12-09T06:15:52,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 299 msec 2024-12-09T06:15:52,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/c1e7cb9253a44c49b6986974227bd7ab is 50, key is test_row_0/A:col10/1733724952021/Put/seqid=0 2024-12-09T06:15:52,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741893_1069 (size=17181) 2024-12-09T06:15:52,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/c1e7cb9253a44c49b6986974227bd7ab 2024-12-09T06:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T06:15:52,048 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:52,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-09T06:15:52,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/34aa8c808fd44653bffc9c64f7e09481 is 50, key is test_row_0/B:col10/1733724952021/Put/seqid=0 2024-12-09T06:15:52,053 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:52,055 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:52,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T06:15:52,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741894_1070 (size=12301) 2024-12-09T06:15:52,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/34aa8c808fd44653bffc9c64f7e09481 2024-12-09T06:15:52,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725012074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725012078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/606ef2b1c73543f2992a85d0ccb69f2d is 50, key is test_row_0/C:col10/1733724952021/Put/seqid=0 2024-12-09T06:15:52,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741895_1071 (size=12301) 2024-12-09T06:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T06:15:52,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725012187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725012189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,195 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a221b4774269407aab459a383c63c35b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a221b4774269407aab459a383c63c35b 2024-12-09T06:15:52,195 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/2dedcf085d3e424ba7734ecba2dd9671 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/2dedcf085d3e424ba7734ecba2dd9671 2024-12-09T06:15:52,206 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into 2dedcf085d3e424ba7734ecba2dd9671(size=12.6 K), total size for store is 22.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:52,206 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:52,206 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=13, startTime=1733724951754; duration=0sec 2024-12-09T06:15:52,206 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into a221b4774269407aab459a383c63c35b(size=12.6 K), total size for store is 22.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:52,206 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:52,206 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=13, startTime=1733724951754; duration=0sec 2024-12-09T06:15:52,207 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:52,207 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:52,207 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:15:52,207 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:52,207 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:52,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T06:15:52,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:52,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,210 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:15:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,210 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:15:52,210 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,210 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/4182bdfabe13404db6446c95a91fb21b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/bc828cc6f6f340abb6cb7e805597b4aa, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/86da9095f4144624ac304ea1eef8919e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/473e5e617d6e419d849544c945942e27] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=43.7 K 2024-12-09T06:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,212 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4182bdfabe13404db6446c95a91fb21b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733724949923 2024-12-09T06:15:52,214 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting bc828cc6f6f340abb6cb7e805597b4aa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733724950269 2024-12-09T06:15:52,219 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 86da9095f4144624ac304ea1eef8919e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733724950965 2024-12-09T06:15:52,220 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 473e5e617d6e419d849544c945942e27, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733724951620 2024-12-09T06:15:52,238 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:52,239 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/527ccfb66acf40a59de49f6ca599b481 is 50, key is test_row_0/C:col10/1733724951620/Put/seqid=0 2024-12-09T06:15:52,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741896_1072 (size=12915) 2024-12-09T06:15:52,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T06:15:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T06:15:52,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725012397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725012398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/606ef2b1c73543f2992a85d0ccb69f2d 2024-12-09T06:15:52,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/c1e7cb9253a44c49b6986974227bd7ab as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/c1e7cb9253a44c49b6986974227bd7ab 2024-12-09T06:15:52,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/c1e7cb9253a44c49b6986974227bd7ab, entries=250, sequenceid=452, filesize=16.8 K 2024-12-09T06:15:52,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/34aa8c808fd44653bffc9c64f7e09481 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/34aa8c808fd44653bffc9c64f7e09481 2024-12-09T06:15:52,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/34aa8c808fd44653bffc9c64f7e09481, entries=150, sequenceid=452, filesize=12.0 K 2024-12-09T06:15:52,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T06:15:52,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:52,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/606ef2b1c73543f2992a85d0ccb69f2d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/606ef2b1c73543f2992a85d0ccb69f2d 2024-12-09T06:15:52,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:52,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/606ef2b1c73543f2992a85d0ccb69f2d, entries=150, sequenceid=452, filesize=12.0 K 2024-12-09T06:15:52,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~231.42 KB/236976, currentSize=100.63 KB/103050 for 89f73a4239a4d6e12de44ad55db0a5e8 in 518ms, sequenceid=452, compaction requested=true 2024-12-09T06:15:52,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:52,542 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:52,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:52,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:52,544 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39919 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:52,544 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:52,544 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,544 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/2dedcf085d3e424ba7734ecba2dd9671, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/47fd9923c33a4cf38404324ed79f7c1a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/c1e7cb9253a44c49b6986974227bd7ab] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=39.0 K 2024-12-09T06:15:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:52,545 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dedcf085d3e424ba7734ecba2dd9671, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733724950965 2024-12-09T06:15:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:15:52,545 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47fd9923c33a4cf38404324ed79f7c1a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733724951620 2024-12-09T06:15:52,546 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1e7cb9253a44c49b6986974227bd7ab, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733724951984 2024-12-09T06:15:52,558 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:52,559 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/5d9368d6b5c5447ca9c5a7fd82836bf9 is 50, key is test_row_0/A:col10/1733724952021/Put/seqid=0 2024-12-09T06:15:52,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741897_1073 (size=12983) 2024-12-09T06:15:52,585 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/5d9368d6b5c5447ca9c5a7fd82836bf9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5d9368d6b5c5447ca9c5a7fd82836bf9 2024-12-09T06:15:52,595 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into 5d9368d6b5c5447ca9c5a7fd82836bf9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:52,595 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:52,595 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=13, startTime=1733724952542; duration=0sec 2024-12-09T06:15:52,595 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:15:52,595 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:52,596 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-09T06:15:52,597 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:15:52,597 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:15:52,597 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. because compaction request was cancelled 2024-12-09T06:15:52,597 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:52,597 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:52,599 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:52,599 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:52,599 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,599 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a221b4774269407aab459a383c63c35b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/64dc10953972409b87715b63ae6fb9f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/34aa8c808fd44653bffc9c64f7e09481] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=34.2 K 2024-12-09T06:15:52,599 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a221b4774269407aab459a383c63c35b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733724950965 2024-12-09T06:15:52,600 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64dc10953972409b87715b63ae6fb9f3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733724951620 2024-12-09T06:15:52,601 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34aa8c808fd44653bffc9c64f7e09481, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733724951984 2024-12-09T06:15:52,613 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:52,614 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a5b593f12380498cbfabad2bb0ac15fb is 50, key is test_row_0/B:col10/1733724952021/Put/seqid=0 2024-12-09T06:15:52,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741898_1074 (size=12983) 2024-12-09T06:15:52,634 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/a5b593f12380498cbfabad2bb0ac15fb as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a5b593f12380498cbfabad2bb0ac15fb 2024-12-09T06:15:52,643 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into a5b593f12380498cbfabad2bb0ac15fb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:52,643 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:52,643 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=13, startTime=1733724952544; duration=0sec 2024-12-09T06:15:52,643 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:52,643 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:52,665 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/527ccfb66acf40a59de49f6ca599b481 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/527ccfb66acf40a59de49f6ca599b481 2024-12-09T06:15:52,676 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into 527ccfb66acf40a59de49f6ca599b481(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:52,676 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:52,676 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=12, startTime=1733724951755; duration=0sec 2024-12-09T06:15:52,676 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:52,676 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:52,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T06:15:52,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,678 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T06:15:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/ffa0fe813006447c9810b94d328a02da is 50, key is test_row_0/A:col10/1733724952072/Put/seqid=0 2024-12-09T06:15:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T06:15:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:52,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:52,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741899_1075 (size=12301) 2024-12-09T06:15:52,729 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/ffa0fe813006447c9810b94d328a02da 2024-12-09T06:15:52,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725012735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725012736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/b4ce7b767eed42c68b091592973f2a91 is 50, key is test_row_0/B:col10/1733724952072/Put/seqid=0 2024-12-09T06:15:52,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741900_1076 (size=12301) 2024-12-09T06:15:52,757 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/b4ce7b767eed42c68b091592973f2a91 2024-12-09T06:15:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/be1becb08d5f484995ceabf43bc0743f is 50, key is test_row_0/C:col10/1733724952072/Put/seqid=0 2024-12-09T06:15:52,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741901_1077 (size=12301) 2024-12-09T06:15:52,782 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/be1becb08d5f484995ceabf43bc0743f 2024-12-09T06:15:52,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/ffa0fe813006447c9810b94d328a02da as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/ffa0fe813006447c9810b94d328a02da 2024-12-09T06:15:52,806 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/ffa0fe813006447c9810b94d328a02da, entries=150, sequenceid=475, filesize=12.0 K 2024-12-09T06:15:52,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/b4ce7b767eed42c68b091592973f2a91 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b4ce7b767eed42c68b091592973f2a91 2024-12-09T06:15:52,814 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b4ce7b767eed42c68b091592973f2a91, entries=150, sequenceid=475, filesize=12.0 K 2024-12-09T06:15:52,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/be1becb08d5f484995ceabf43bc0743f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/be1becb08d5f484995ceabf43bc0743f 2024-12-09T06:15:52,823 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/be1becb08d5f484995ceabf43bc0743f, entries=150, sequenceid=475, filesize=12.0 K 2024-12-09T06:15:52,824 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 89f73a4239a4d6e12de44ad55db0a5e8 in 146ms, sequenceid=475, compaction requested=true 2024-12-09T06:15:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T06:15:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-09T06:15:52,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T06:15:52,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 772 msec 2024-12-09T06:15:52,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 780 msec 2024-12-09T06:15:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:52,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T06:15:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:52,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/11da901592334ef89d4b665c165aa22a is 50, key is test_row_0/A:col10/1733724952727/Put/seqid=0 2024-12-09T06:15:52,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741902_1078 (size=14741) 2024-12-09T06:15:52,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725012891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725012892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725013007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725013007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T06:15:53,197 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:53,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-09T06:15:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T06:15:53,199 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:53,200 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:53,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:53,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725013217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725013217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/11da901592334ef89d4b665c165aa22a 2024-12-09T06:15:53,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/9b5098e07c74454c97437f913302c426 is 50, key is test_row_0/B:col10/1733724952727/Put/seqid=0 2024-12-09T06:15:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741903_1079 (size=12301) 2024-12-09T06:15:53,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/9b5098e07c74454c97437f913302c426 2024-12-09T06:15:53,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/6cb5e9b4c95d492190c7e849cb6597c7 is 50, key is test_row_0/C:col10/1733724952727/Put/seqid=0 2024-12-09T06:15:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T06:15:53,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741904_1080 (size=12301) 2024-12-09T06:15:53,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/6cb5e9b4c95d492190c7e849cb6597c7 2024-12-09T06:15:53,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/11da901592334ef89d4b665c165aa22a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/11da901592334ef89d4b665c165aa22a 2024-12-09T06:15:53,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/11da901592334ef89d4b665c165aa22a, entries=200, sequenceid=490, filesize=14.4 K 2024-12-09T06:15:53,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/9b5098e07c74454c97437f913302c426 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9b5098e07c74454c97437f913302c426 2024-12-09T06:15:53,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9b5098e07c74454c97437f913302c426, entries=150, sequenceid=490, filesize=12.0 K 2024-12-09T06:15:53,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/6cb5e9b4c95d492190c7e849cb6597c7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6cb5e9b4c95d492190c7e849cb6597c7 2024-12-09T06:15:53,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6cb5e9b4c95d492190c7e849cb6597c7, entries=150, sequenceid=490, filesize=12.0 K 2024-12-09T06:15:53,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 89f73a4239a4d6e12de44ad55db0a5e8 in 486ms, sequenceid=490, compaction requested=true 2024-12-09T06:15:53,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:53,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:15:53,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:53,334 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:53,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:53,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:53,334 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:53,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:53,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:53,335 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:53,335 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:53,335 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:53,335 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:53,335 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:53,335 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:53,335 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a5b593f12380498cbfabad2bb0ac15fb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b4ce7b767eed42c68b091592973f2a91, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9b5098e07c74454c97437f913302c426] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=36.7 K 2024-12-09T06:15:53,335 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5d9368d6b5c5447ca9c5a7fd82836bf9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/ffa0fe813006447c9810b94d328a02da, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/11da901592334ef89d4b665c165aa22a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=39.1 K 2024-12-09T06:15:53,336 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b593f12380498cbfabad2bb0ac15fb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733724951984 2024-12-09T06:15:53,336 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d9368d6b5c5447ca9c5a7fd82836bf9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733724951984 2024-12-09T06:15:53,336 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffa0fe813006447c9810b94d328a02da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733724952058 2024-12-09T06:15:53,336 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ce7b767eed42c68b091592973f2a91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733724952058 2024-12-09T06:15:53,337 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b5098e07c74454c97437f913302c426, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733724952717 2024-12-09T06:15:53,337 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11da901592334ef89d4b665c165aa22a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733724952717 2024-12-09T06:15:53,350 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:53,350 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/182ce5e7fba54da5953b7c43e76a5176 is 50, key is test_row_0/B:col10/1733724952727/Put/seqid=0 2024-12-09T06:15:53,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T06:15:53,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:53,353 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T06:15:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:53,362 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:53,363 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a4719abb79e146b5b5ea41e15a0fa05c is 50, key is test_row_0/A:col10/1733724952727/Put/seqid=0 2024-12-09T06:15:53,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/bdb3f3a4968c46ff8350ab9813588720 is 50, key is test_row_0/A:col10/1733724952889/Put/seqid=0 2024-12-09T06:15:53,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741905_1081 (size=13085) 2024-12-09T06:15:53,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741906_1082 (size=13085) 2024-12-09T06:15:53,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741907_1083 (size=12301) 2024-12-09T06:15:53,406 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a4719abb79e146b5b5ea41e15a0fa05c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a4719abb79e146b5b5ea41e15a0fa05c 2024-12-09T06:15:53,415 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into a4719abb79e146b5b5ea41e15a0fa05c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:53,415 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:53,415 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=13, startTime=1733724953334; duration=0sec 2024-12-09T06:15:53,416 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:53,416 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:53,416 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:15:53,418 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:15:53,419 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:15:53,419 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:53,419 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/527ccfb66acf40a59de49f6ca599b481, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/606ef2b1c73543f2992a85d0ccb69f2d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/be1becb08d5f484995ceabf43bc0743f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6cb5e9b4c95d492190c7e849cb6597c7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=48.7 K 2024-12-09T06:15:53,420 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 527ccfb66acf40a59de49f6ca599b481, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733724950965 2024-12-09T06:15:53,421 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 606ef2b1c73543f2992a85d0ccb69f2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733724951984 2024-12-09T06:15:53,421 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting be1becb08d5f484995ceabf43bc0743f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733724952058 2024-12-09T06:15:53,422 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb5e9b4c95d492190c7e849cb6597c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733724952717 2024-12-09T06:15:53,446 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:53,447 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/63e0f95edb144787a8f138fc9c1a20cd is 50, key is test_row_0/C:col10/1733724952727/Put/seqid=0 2024-12-09T06:15:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741908_1084 (size=13051) 2024-12-09T06:15:53,478 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/63e0f95edb144787a8f138fc9c1a20cd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/63e0f95edb144787a8f138fc9c1a20cd 2024-12-09T06:15:53,488 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into 63e0f95edb144787a8f138fc9c1a20cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:53,488 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:53,488 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=12, startTime=1733724953334; duration=0sec 2024-12-09T06:15:53,488 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:53,488 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T06:15:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:53,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:53,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725013566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725013566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725013676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725013676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,799 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/bdb3f3a4968c46ff8350ab9813588720 2024-12-09T06:15:53,799 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/182ce5e7fba54da5953b7c43e76a5176 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/182ce5e7fba54da5953b7c43e76a5176 2024-12-09T06:15:53,808 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into 182ce5e7fba54da5953b7c43e76a5176(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:53,808 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:53,808 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=13, startTime=1733724953334; duration=0sec 2024-12-09T06:15:53,808 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3fe2901a8bba417f9920c75c4b07ca51 is 50, key is test_row_0/B:col10/1733724952889/Put/seqid=0 2024-12-09T06:15:53,808 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:53,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741909_1085 (size=12301) 2024-12-09T06:15:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T06:15:53,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725013886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725013886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:54,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725014197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725014197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:54,227 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3fe2901a8bba417f9920c75c4b07ca51 2024-12-09T06:15:54,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e06aba88d0d84b9ca20482eb3f290d4c is 50, key is test_row_0/C:col10/1733724952889/Put/seqid=0 2024-12-09T06:15:54,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741910_1086 (size=12301) 2024-12-09T06:15:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T06:15:54,647 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e06aba88d0d84b9ca20482eb3f290d4c 2024-12-09T06:15:54,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/bdb3f3a4968c46ff8350ab9813588720 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bdb3f3a4968c46ff8350ab9813588720 2024-12-09T06:15:54,658 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bdb3f3a4968c46ff8350ab9813588720, entries=150, sequenceid=507, filesize=12.0 K 2024-12-09T06:15:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3fe2901a8bba417f9920c75c4b07ca51 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3fe2901a8bba417f9920c75c4b07ca51 2024-12-09T06:15:54,664 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3fe2901a8bba417f9920c75c4b07ca51, entries=150, sequenceid=507, filesize=12.0 K 2024-12-09T06:15:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/e06aba88d0d84b9ca20482eb3f290d4c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e06aba88d0d84b9ca20482eb3f290d4c 2024-12-09T06:15:54,671 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e06aba88d0d84b9ca20482eb3f290d4c, entries=150, sequenceid=507, filesize=12.0 K 2024-12-09T06:15:54,672 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 89f73a4239a4d6e12de44ad55db0a5e8 in 1320ms, sequenceid=507, compaction requested=false 2024-12-09T06:15:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T06:15:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-09T06:15:54,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T06:15:54,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-12-09T06:15:54,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4790 sec 2024-12-09T06:15:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:54,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T06:15:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:54,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/f28eff85d36d4538a6730057b3d36d57 is 50, key is test_row_0/A:col10/1733724953564/Put/seqid=0 2024-12-09T06:15:54,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741911_1087 (size=14741) 2024-12-09T06:15:54,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725014733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725014734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:54,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725014846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725014846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50344 deadline: 1733725014996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,001 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50306 deadline: 1733725015027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50324 deadline: 1733725015027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,029 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,029 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 89f73a4239a4d6e12de44ad55db0a5e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725015057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725015057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/f28eff85d36d4538a6730057b3d36d57 2024-12-09T06:15:55,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/d8d64e1155224be9b50664585ab4bd38 is 50, key is test_row_0/B:col10/1733724953564/Put/seqid=0 2024-12-09T06:15:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741912_1088 (size=12301) 2024-12-09T06:15:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T06:15:55,347 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:55,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-09T06:15:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T06:15:55,350 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:55,350 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:55,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:55,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725015367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725015367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T06:15:55,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T06:15:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/d8d64e1155224be9b50664585ab4bd38 2024-12-09T06:15:55,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/37ecec39268e4ab085634e94274563a1 is 50, key is test_row_0/C:col10/1733724953564/Put/seqid=0 2024-12-09T06:15:55,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741913_1089 (size=12301) 2024-12-09T06:15:55,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T06:15:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T06:15:55,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T06:15:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725015877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725015877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:55,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/37ecec39268e4ab085634e94274563a1 2024-12-09T06:15:55,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/f28eff85d36d4538a6730057b3d36d57 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f28eff85d36d4538a6730057b3d36d57 2024-12-09T06:15:55,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T06:15:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:55,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f28eff85d36d4538a6730057b3d36d57, entries=200, sequenceid=527, filesize=14.4 K 2024-12-09T06:15:55,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/d8d64e1155224be9b50664585ab4bd38 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/d8d64e1155224be9b50664585ab4bd38 2024-12-09T06:15:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T06:15:55,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/d8d64e1155224be9b50664585ab4bd38, entries=150, sequenceid=527, filesize=12.0 K 2024-12-09T06:15:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/37ecec39268e4ab085634e94274563a1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/37ecec39268e4ab085634e94274563a1 2024-12-09T06:15:55,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/37ecec39268e4ab085634e94274563a1, entries=150, sequenceid=527, filesize=12.0 K 2024-12-09T06:15:55,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 89f73a4239a4d6e12de44ad55db0a5e8 in 1280ms, sequenceid=527, compaction requested=true 2024-12-09T06:15:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:55,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:15:55,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:55,989 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:55,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:55,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:55,989 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:55,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:55,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:55,990 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:55,990 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:55,990 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:55,990 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:55,990 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,990 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:55,990 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a4719abb79e146b5b5ea41e15a0fa05c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bdb3f3a4968c46ff8350ab9813588720, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f28eff85d36d4538a6730057b3d36d57] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=39.2 K 2024-12-09T06:15:55,990 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/182ce5e7fba54da5953b7c43e76a5176, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3fe2901a8bba417f9920c75c4b07ca51, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/d8d64e1155224be9b50664585ab4bd38] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=36.8 K 2024-12-09T06:15:55,991 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4719abb79e146b5b5ea41e15a0fa05c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733724952717 2024-12-09T06:15:55,991 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 182ce5e7fba54da5953b7c43e76a5176, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733724952717 2024-12-09T06:15:55,991 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb3f3a4968c46ff8350ab9813588720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733724952868 2024-12-09T06:15:55,991 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fe2901a8bba417f9920c75c4b07ca51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733724952868 2024-12-09T06:15:55,992 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting f28eff85d36d4538a6730057b3d36d57, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733724953557 2024-12-09T06:15:55,992 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting d8d64e1155224be9b50664585ab4bd38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733724953557 2024-12-09T06:15:56,000 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:56,000 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:56,001 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/67e137bd64404a8b909410df7da5a95b is 50, key is test_row_0/B:col10/1733724953564/Put/seqid=0 2024-12-09T06:15:56,001 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f is 50, key is test_row_0/A:col10/1733724953564/Put/seqid=0 2024-12-09T06:15:56,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741914_1090 (size=13187) 2024-12-09T06:15:56,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741915_1091 (size=13187) 2024-12-09T06:15:56,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T06:15:56,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:56,124 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T06:15:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:56,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:56,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a3683e89a9574ab88f2d1b5c2495fbc4 is 50, key is test_row_0/A:col10/1733724954732/Put/seqid=0 2024-12-09T06:15:56,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741916_1092 (size=12301) 2024-12-09T06:15:56,413 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/67e137bd64404a8b909410df7da5a95b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/67e137bd64404a8b909410df7da5a95b 2024-12-09T06:15:56,418 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f 2024-12-09T06:15:56,419 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into 67e137bd64404a8b909410df7da5a95b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:56,419 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:56,419 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=13, startTime=1733724955989; duration=0sec 2024-12-09T06:15:56,419 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:56,419 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:56,419 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:56,420 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:56,420 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:15:56,421 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:56,421 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/63e0f95edb144787a8f138fc9c1a20cd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e06aba88d0d84b9ca20482eb3f290d4c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/37ecec39268e4ab085634e94274563a1] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=36.8 K 2024-12-09T06:15:56,421 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 63e0f95edb144787a8f138fc9c1a20cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733724952717 2024-12-09T06:15:56,422 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e06aba88d0d84b9ca20482eb3f290d4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733724952868 2024-12-09T06:15:56,422 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ecec39268e4ab085634e94274563a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733724953557 2024-12-09T06:15:56,424 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into 7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:56,424 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:56,424 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=13, startTime=1733724955989; duration=0sec 2024-12-09T06:15:56,424 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:56,424 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:56,431 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:56,431 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/a9b510d6fac34529b7ac9f942bd96a36 is 50, key is test_row_0/C:col10/1733724953564/Put/seqid=0 2024-12-09T06:15:56,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741917_1093 (size=13153) 2024-12-09T06:15:56,441 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/a9b510d6fac34529b7ac9f942bd96a36 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a9b510d6fac34529b7ac9f942bd96a36 2024-12-09T06:15:56,448 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into a9b510d6fac34529b7ac9f942bd96a36(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:56,448 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:56,449 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=13, startTime=1733724955989; duration=0sec 2024-12-09T06:15:56,449 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:56,449 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T06:15:56,541 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a3683e89a9574ab88f2d1b5c2495fbc4 2024-12-09T06:15:56,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/86423159560a448a908b00cd2f74213c is 50, key is test_row_0/B:col10/1733724954732/Put/seqid=0 2024-12-09T06:15:56,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741918_1094 (size=12301) 2024-12-09T06:15:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:56,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:56,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725016916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:56,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:56,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:56,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:56,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725016916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:56,962 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/86423159560a448a908b00cd2f74213c 2024-12-09T06:15:56,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/96f3f3678a82471c958bf4c368935afe is 50, key is test_row_0/C:col10/1733724954732/Put/seqid=0 2024-12-09T06:15:56,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741919_1095 (size=12301) 2024-12-09T06:15:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725017027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725017026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725017237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725017237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,384 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/96f3f3678a82471c958bf4c368935afe 2024-12-09T06:15:57,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a3683e89a9574ab88f2d1b5c2495fbc4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a3683e89a9574ab88f2d1b5c2495fbc4 2024-12-09T06:15:57,407 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a3683e89a9574ab88f2d1b5c2495fbc4, entries=150, sequenceid=542, filesize=12.0 K 2024-12-09T06:15:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/86423159560a448a908b00cd2f74213c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/86423159560a448a908b00cd2f74213c 2024-12-09T06:15:57,419 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/86423159560a448a908b00cd2f74213c, entries=150, sequenceid=542, filesize=12.0 K 2024-12-09T06:15:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/96f3f3678a82471c958bf4c368935afe as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/96f3f3678a82471c958bf4c368935afe 2024-12-09T06:15:57,446 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/96f3f3678a82471c958bf4c368935afe, entries=150, sequenceid=542, filesize=12.0 K 2024-12-09T06:15:57,447 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 89f73a4239a4d6e12de44ad55db0a5e8 in 1323ms, sequenceid=542, compaction requested=false 2024-12-09T06:15:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-09T06:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-09T06:15:57,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T06:15:57,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0970 sec 2024-12-09T06:15:57,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1040 sec 2024-12-09T06:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T06:15:57,497 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:57,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-09T06:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T06:15:57,501 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:57,502 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:57,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:57,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T06:15:57,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:57,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:57,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:57,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:57,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:57,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:57,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/131dc694c4464007909db5f482ca8fd3 is 50, key is test_row_0/A:col10/1733724957547/Put/seqid=0 2024-12-09T06:15:57,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741920_1096 (size=12301) 2024-12-09T06:15:57,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/131dc694c4464007909db5f482ca8fd3 2024-12-09T06:15:57,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/ef8aac5070474396bfb4a3a02ef6ec5d is 50, key is test_row_0/B:col10/1733724957547/Put/seqid=0 2024-12-09T06:15:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T06:15:57,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741921_1097 (size=12301) 2024-12-09T06:15:57,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725017625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725017629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T06:15:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:57,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:57,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725017736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725017736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T06:15:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T06:15:57,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:57,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:57,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:57,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:57,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:57,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725017947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725017949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:57,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T06:15:57,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:57,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:57,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:57,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:57,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:15:58,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/ef8aac5070474396bfb4a3a02ef6ec5d 2024-12-09T06:15:58,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/7b1a3e86cd1b460d90dedffb7ba78d87 is 50, key is test_row_0/C:col10/1733724957547/Put/seqid=0 2024-12-09T06:15:58,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741922_1098 (size=12301) 2024-12-09T06:15:58,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/7b1a3e86cd1b460d90dedffb7ba78d87 2024-12-09T06:15:58,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/131dc694c4464007909db5f482ca8fd3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/131dc694c4464007909db5f482ca8fd3 2024-12-09T06:15:58,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/131dc694c4464007909db5f482ca8fd3, entries=150, sequenceid=564, filesize=12.0 K 2024-12-09T06:15:58,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/ef8aac5070474396bfb4a3a02ef6ec5d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ef8aac5070474396bfb4a3a02ef6ec5d 2024-12-09T06:15:58,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ef8aac5070474396bfb4a3a02ef6ec5d, entries=150, sequenceid=564, filesize=12.0 K 2024-12-09T06:15:58,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/7b1a3e86cd1b460d90dedffb7ba78d87 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/7b1a3e86cd1b460d90dedffb7ba78d87 2024-12-09T06:15:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T06:15:58,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/7b1a3e86cd1b460d90dedffb7ba78d87, entries=150, sequenceid=564, filesize=12.0 K 2024-12-09T06:15:58,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T06:15:58,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 89f73a4239a4d6e12de44ad55db0a5e8 in 581ms, sequenceid=564, compaction requested=true 2024-12-09T06:15:58,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:58,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:58,130 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T06:15:58,130 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:58,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:58,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:58,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:58,132 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:58,132 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:15:58,133 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:58,133 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a3683e89a9574ab88f2d1b5c2495fbc4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/131dc694c4464007909db5f482ca8fd3] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=36.9 K 2024-12-09T06:15:58,134 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733724953557 2024-12-09T06:15:58,134 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3683e89a9574ab88f2d1b5c2495fbc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733724954724 2024-12-09T06:15:58,134 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 131dc694c4464007909db5f482ca8fd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733724956911 2024-12-09T06:15:58,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:15:58,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:58,138 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:58,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:15:58,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:58,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:15:58,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:58,139 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:58,139 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:15:58,139 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:58,140 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/67e137bd64404a8b909410df7da5a95b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/86423159560a448a908b00cd2f74213c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ef8aac5070474396bfb4a3a02ef6ec5d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=36.9 K 2024-12-09T06:15:58,141 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e137bd64404a8b909410df7da5a95b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733724953557 2024-12-09T06:15:58,145 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 86423159560a448a908b00cd2f74213c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733724954724 2024-12-09T06:15:58,146 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ef8aac5070474396bfb4a3a02ef6ec5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733724956911 2024-12-09T06:15:58,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a7090bd3c83847cd9ba3f3fece7abb2f is 50, key is test_row_0/A:col10/1733724957558/Put/seqid=0 2024-12-09T06:15:58,155 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:58,156 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/06b7035808694a26a263718a1f1b062b is 50, key is test_row_0/A:col10/1733724957547/Put/seqid=0 2024-12-09T06:15:58,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741923_1099 (size=12301) 2024-12-09T06:15:58,175 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:58,176 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/5767ecde20b741b5832999d1001e62d6 is 50, key is test_row_0/B:col10/1733724957547/Put/seqid=0 2024-12-09T06:15:58,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741924_1100 (size=13289) 2024-12-09T06:15:58,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741925_1101 (size=13289) 2024-12-09T06:15:58,192 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/5767ecde20b741b5832999d1001e62d6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5767ecde20b741b5832999d1001e62d6 2024-12-09T06:15:58,206 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into 5767ecde20b741b5832999d1001e62d6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:58,206 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:58,206 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=13, startTime=1733724958138; duration=0sec 2024-12-09T06:15:58,206 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:15:58,206 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:15:58,206 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:15:58,209 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:15:58,209 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:15:58,209 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:58,209 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a9b510d6fac34529b7ac9f942bd96a36, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/96f3f3678a82471c958bf4c368935afe, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/7b1a3e86cd1b460d90dedffb7ba78d87] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=36.9 K 2024-12-09T06:15:58,209 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b510d6fac34529b7ac9f942bd96a36, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733724953557 2024-12-09T06:15:58,210 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f3f3678a82471c958bf4c368935afe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733724954724 2024-12-09T06:15:58,210 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b1a3e86cd1b460d90dedffb7ba78d87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733724956911 2024-12-09T06:15:58,220 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T06:15:58,229 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:15:58,230 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/b1ab2bd80fde41c1902f29efff0761aa is 50, key is test_row_0/C:col10/1733724957547/Put/seqid=0 2024-12-09T06:15:58,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741926_1102 (size=13255) 2024-12-09T06:15:58,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:15:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:15:58,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725018309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725018314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725018418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:58,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725018426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:58,571 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a7090bd3c83847cd9ba3f3fece7abb2f 2024-12-09T06:15:58,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/fd6cc17353644b4d83e40a2c30037895 is 50, key is test_row_0/B:col10/1733724957558/Put/seqid=0 2024-12-09T06:15:58,601 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/06b7035808694a26a263718a1f1b062b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/06b7035808694a26a263718a1f1b062b 2024-12-09T06:15:58,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741927_1103 (size=12301) 2024-12-09T06:15:58,628 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into 06b7035808694a26a263718a1f1b062b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:58,629 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:58,629 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=13, startTime=1733724958130; duration=0sec 2024-12-09T06:15:58,629 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:58,629 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:15:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725018626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T06:15:58,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725018636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:58,669 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/b1ab2bd80fde41c1902f29efff0761aa as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b1ab2bd80fde41c1902f29efff0761aa 2024-12-09T06:15:58,679 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into b1ab2bd80fde41c1902f29efff0761aa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:15:58,679 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:58,679 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=13, startTime=1733724958138; duration=0sec 2024-12-09T06:15:58,680 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:15:58,680 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:15:58,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725018936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:58,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725018956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:59,015 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/fd6cc17353644b4d83e40a2c30037895 2024-12-09T06:15:59,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/444c43b20f1540b0bd8ecdbc33fb28a2 is 50, key is test_row_0/C:col10/1733724957558/Put/seqid=0 2024-12-09T06:15:59,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741928_1104 (size=12301) 2024-12-09T06:15:59,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725019447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:59,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:59,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:59,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:59,462 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/444c43b20f1540b0bd8ecdbc33fb28a2 2024-12-09T06:15:59,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:15:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725019468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:15:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a7090bd3c83847cd9ba3f3fece7abb2f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a7090bd3c83847cd9ba3f3fece7abb2f 2024-12-09T06:15:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:15:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:15:59,477 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a7090bd3c83847cd9ba3f3fece7abb2f, entries=150, sequenceid=577, filesize=12.0 K 2024-12-09T06:15:59,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/fd6cc17353644b4d83e40a2c30037895 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/fd6cc17353644b4d83e40a2c30037895 2024-12-09T06:15:59,491 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/fd6cc17353644b4d83e40a2c30037895, entries=150, sequenceid=577, filesize=12.0 K 2024-12-09T06:15:59,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/444c43b20f1540b0bd8ecdbc33fb28a2 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/444c43b20f1540b0bd8ecdbc33fb28a2 2024-12-09T06:15:59,503 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/444c43b20f1540b0bd8ecdbc33fb28a2, entries=150, sequenceid=577, filesize=12.0 K 2024-12-09T06:15:59,504 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 89f73a4239a4d6e12de44ad55db0a5e8 in 1374ms, sequenceid=577, compaction requested=false 2024-12-09T06:15:59,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:15:59,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:59,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-09T06:15:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-09T06:15:59,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T06:15:59,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-12-09T06:15:59,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0170 sec 2024-12-09T06:15:59,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T06:15:59,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:15:59,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:15:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-09T06:15:59,650 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:15:59,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T06:15:59,653 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:15:59,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:15:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T06:15:59,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T06:15:59,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:15:59,807 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T06:15:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:15:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:15:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:15:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:15:59,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/e0abece07f1f4b7998332ebc5168d1eb is 50, key is test_row_0/A:col10/1733724958284/Put/seqid=0 2024-12-09T06:15:59,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741929_1105 (size=12301) 2024-12-09T06:15:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T06:16:00,249 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/e0abece07f1f4b7998332ebc5168d1eb 2024-12-09T06:16:00,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/9caa2efd7bb0449c8d26f8c0005efd3b is 50, key is test_row_0/B:col10/1733724958284/Put/seqid=0 2024-12-09T06:16:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T06:16:00,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741930_1106 (size=12301) 2024-12-09T06:16:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:16:00,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. as already flushing 2024-12-09T06:16:00,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725020500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:00,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725020509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:00,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725020606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725020626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:00,704 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/9caa2efd7bb0449c8d26f8c0005efd3b 2024-12-09T06:16:00,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/a13c60ea147b42e99be12215160727a2 is 50, key is test_row_0/C:col10/1733724958284/Put/seqid=0 2024-12-09T06:16:00,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741931_1107 (size=12301) 2024-12-09T06:16:00,769 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/a13c60ea147b42e99be12215160727a2 2024-12-09T06:16:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/e0abece07f1f4b7998332ebc5168d1eb as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/e0abece07f1f4b7998332ebc5168d1eb 2024-12-09T06:16:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T06:16:00,789 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/e0abece07f1f4b7998332ebc5168d1eb, entries=150, sequenceid=599, filesize=12.0 K 2024-12-09T06:16:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/9caa2efd7bb0449c8d26f8c0005efd3b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9caa2efd7bb0449c8d26f8c0005efd3b 2024-12-09T06:16:00,813 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9caa2efd7bb0449c8d26f8c0005efd3b, entries=150, sequenceid=599, filesize=12.0 K 2024-12-09T06:16:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/a13c60ea147b42e99be12215160727a2 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a13c60ea147b42e99be12215160727a2 2024-12-09T06:16:00,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725020817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:00,822 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a13c60ea147b42e99be12215160727a2, entries=150, sequenceid=599, filesize=12.0 K 2024-12-09T06:16:00,833 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 89f73a4239a4d6e12de44ad55db0a5e8 in 1025ms, sequenceid=599, compaction requested=true 2024-12-09T06:16:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:16:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:16:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-09T06:16:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-09T06:16:00,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T06:16:00,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1890 sec 2024-12-09T06:16:00,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.1990 sec 2024-12-09T06:16:00,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T06:16:00,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:16:00,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:00,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:16:00,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:00,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:16:00,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:16:00,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/77651f70f1274d879cebc958158dadab is 50, key is test_row_0/A:col10/1733724960461/Put/seqid=0 2024-12-09T06:16:00,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741932_1108 (size=14741) 2024-12-09T06:16:00,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/77651f70f1274d879cebc958158dadab 2024-12-09T06:16:00,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/8e7a0cda90f144aa8e19bba3a552265b is 50, key is test_row_0/B:col10/1733724960461/Put/seqid=0 2024-12-09T06:16:00,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725020944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:00,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741933_1109 (size=12301) 2024-12-09T06:16:00,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/8e7a0cda90f144aa8e19bba3a552265b 2024-12-09T06:16:01,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/58607def505e4e51bf6226cd011fa891 is 50, key is test_row_0/C:col10/1733724960461/Put/seqid=0 2024-12-09T06:16:01,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741934_1110 (size=12301) 2024-12-09T06:16:01,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725021057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:01,063 INFO [Thread-155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-155. 2024-12-09T06:16:01,063 DEBUG [Thread-155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:01,064 INFO [Thread-159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-159. 2024-12-09T06:16:01,064 DEBUG [Thread-159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:01,065 INFO [Thread-157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-157. 2024-12-09T06:16:01,065 DEBUG [Thread-157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:01,069 INFO [Thread-161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-161. 2024-12-09T06:16:01,069 DEBUG [Thread-161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:01,070 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,070 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,071 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:01,071 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,071 DEBUG [Thread-161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,071 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:01,071 DEBUG [Thread-161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:01,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:01,073 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50326 deadline: 1733725021127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:01,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50382 deadline: 1733725021266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:01,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:01,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/58607def505e4e51bf6226cd011fa891 2024-12-09T06:16:01,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/77651f70f1274d879cebc958158dadab as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/77651f70f1274d879cebc958158dadab 2024-12-09T06:16:01,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/77651f70f1274d879cebc958158dadab, entries=200, sequenceid=614, filesize=14.4 K 2024-12-09T06:16:01,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/8e7a0cda90f144aa8e19bba3a552265b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/8e7a0cda90f144aa8e19bba3a552265b 2024-12-09T06:16:01,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/8e7a0cda90f144aa8e19bba3a552265b, entries=150, sequenceid=614, filesize=12.0 K 2024-12-09T06:16:01,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/58607def505e4e51bf6226cd011fa891 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/58607def505e4e51bf6226cd011fa891 2024-12-09T06:16:01,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/58607def505e4e51bf6226cd011fa891, entries=150, sequenceid=614, filesize=12.0 K 2024-12-09T06:16:01,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 89f73a4239a4d6e12de44ad55db0a5e8 in 581ms, sequenceid=614, compaction requested=true 2024-12-09T06:16:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:16:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:01,432 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:01,432 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:01,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 89f73a4239a4d6e12de44ad55db0a5e8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:01,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:01,434 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:01,434 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:01,434 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/B is initiating minor compaction (all files) 2024-12-09T06:16:01,434 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/A is initiating minor compaction (all files) 2024-12-09T06:16:01,434 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/B in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:16:01,434 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/A in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:16:01,434 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5767ecde20b741b5832999d1001e62d6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/fd6cc17353644b4d83e40a2c30037895, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9caa2efd7bb0449c8d26f8c0005efd3b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/8e7a0cda90f144aa8e19bba3a552265b] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=49.0 K 2024-12-09T06:16:01,434 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/06b7035808694a26a263718a1f1b062b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a7090bd3c83847cd9ba3f3fece7abb2f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/e0abece07f1f4b7998332ebc5168d1eb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/77651f70f1274d879cebc958158dadab] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=51.4 K 2024-12-09T06:16:01,434 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5767ecde20b741b5832999d1001e62d6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733724956911 2024-12-09T06:16:01,434 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b7035808694a26a263718a1f1b062b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733724956911 2024-12-09T06:16:01,434 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting fd6cc17353644b4d83e40a2c30037895, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733724957558 2024-12-09T06:16:01,434 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7090bd3c83847cd9ba3f3fece7abb2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733724957558 2024-12-09T06:16:01,435 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9caa2efd7bb0449c8d26f8c0005efd3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733724958284 2024-12-09T06:16:01,435 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0abece07f1f4b7998332ebc5168d1eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733724958284 2024-12-09T06:16:01,435 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7a0cda90f144aa8e19bba3a552265b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1733724960461 2024-12-09T06:16:01,435 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77651f70f1274d879cebc958158dadab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1733724960461 2024-12-09T06:16:01,444 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#A#compaction#98 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:01,444 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a78b2f28dd1041efb07928c88ddfc29d is 50, key is test_row_0/A:col10/1733724960461/Put/seqid=0 2024-12-09T06:16:01,445 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:01,445 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/1b6a36741b2247359e7e97d095aa6e54 is 50, key is test_row_0/B:col10/1733724960461/Put/seqid=0 2024-12-09T06:16:01,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741935_1111 (size=13425) 2024-12-09T06:16:01,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741936_1112 (size=13425) 2024-12-09T06:16:01,461 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/1b6a36741b2247359e7e97d095aa6e54 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/1b6a36741b2247359e7e97d095aa6e54 2024-12-09T06:16:01,461 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/a78b2f28dd1041efb07928c88ddfc29d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a78b2f28dd1041efb07928c88ddfc29d 2024-12-09T06:16:01,468 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/B of 89f73a4239a4d6e12de44ad55db0a5e8 into 1b6a36741b2247359e7e97d095aa6e54(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:01,468 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/A of 89f73a4239a4d6e12de44ad55db0a5e8 into a78b2f28dd1041efb07928c88ddfc29d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:01,468 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:16:01,468 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:16:01,468 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/A, priority=12, startTime=1733724961432; duration=0sec 2024-12-09T06:16:01,468 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/B, priority=12, startTime=1733724961432; duration=0sec 2024-12-09T06:16:01,468 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:01,468 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:01,468 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:B 2024-12-09T06:16:01,468 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:A 2024-12-09T06:16:01,468 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:01,470 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:01,470 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 89f73a4239a4d6e12de44ad55db0a5e8/C is initiating minor compaction (all files) 2024-12-09T06:16:01,470 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 89f73a4239a4d6e12de44ad55db0a5e8/C in TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:16:01,470 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b1ab2bd80fde41c1902f29efff0761aa, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/444c43b20f1540b0bd8ecdbc33fb28a2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a13c60ea147b42e99be12215160727a2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/58607def505e4e51bf6226cd011fa891] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp, totalSize=49.0 K 2024-12-09T06:16:01,471 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b1ab2bd80fde41c1902f29efff0761aa, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733724956911 2024-12-09T06:16:01,471 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 444c43b20f1540b0bd8ecdbc33fb28a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733724957558 2024-12-09T06:16:01,471 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a13c60ea147b42e99be12215160727a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733724958284 2024-12-09T06:16:01,471 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 58607def505e4e51bf6226cd011fa891, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1733724960461 2024-12-09T06:16:01,482 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 89f73a4239a4d6e12de44ad55db0a5e8#C#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:01,482 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/6b2b31e61e8843769a1386b2e350fcb8 is 50, key is test_row_0/C:col10/1733724960461/Put/seqid=0 2024-12-09T06:16:01,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741937_1113 (size=13391) 2024-12-09T06:16:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:16:01,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T06:16:01,578 INFO [Thread-152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-152. 2024-12-09T06:16:01,578 DEBUG [Thread-152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:01,578 DEBUG [Thread-152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:16:01,578 DEBUG [Thread-152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:01,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:01,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:16:01,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:01,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:16:01,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:01,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/5920da35d7c541109c6b0d7eb44cefcb is 50, key is test_row_0/A:col10/1733724961577/Put/seqid=0 2024-12-09T06:16:01,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741938_1114 (size=12301) 2024-12-09T06:16:01,637 INFO [Thread-148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-148. 2024-12-09T06:16:01,637 DEBUG [Thread-148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:01,638 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,638 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:01,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T06:16:01,797 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:01,893 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/6b2b31e61e8843769a1386b2e350fcb8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6b2b31e61e8843769a1386b2e350fcb8 2024-12-09T06:16:01,898 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 89f73a4239a4d6e12de44ad55db0a5e8/C of 89f73a4239a4d6e12de44ad55db0a5e8 into 6b2b31e61e8843769a1386b2e350fcb8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:01,898 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:16:01,898 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., storeName=89f73a4239a4d6e12de44ad55db0a5e8/C, priority=12, startTime=1733724961432; duration=0sec 2024-12-09T06:16:01,898 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:01,898 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 89f73a4239a4d6e12de44ad55db0a5e8:C 2024-12-09T06:16:01,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/5920da35d7c541109c6b0d7eb44cefcb 2024-12-09T06:16:01,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3369d5380cd74808a9a17a096877572d is 50, key is test_row_0/B:col10/1733724961577/Put/seqid=0 2024-12-09T06:16:02,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741939_1115 (size=12301) 2024-12-09T06:16:02,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3369d5380cd74808a9a17a096877572d 2024-12-09T06:16:02,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/cf8dc57fa8f243a69a1c9297b041dd1e is 50, key is test_row_0/C:col10/1733724961577/Put/seqid=0 2024-12-09T06:16:02,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741940_1116 (size=12301) 2024-12-09T06:16:02,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/cf8dc57fa8f243a69a1c9297b041dd1e 2024-12-09T06:16:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/5920da35d7c541109c6b0d7eb44cefcb as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5920da35d7c541109c6b0d7eb44cefcb 2024-12-09T06:16:02,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5920da35d7c541109c6b0d7eb44cefcb, entries=150, sequenceid=634, filesize=12.0 K 2024-12-09T06:16:02,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/3369d5380cd74808a9a17a096877572d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3369d5380cd74808a9a17a096877572d 2024-12-09T06:16:02,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3369d5380cd74808a9a17a096877572d, entries=150, sequenceid=634, filesize=12.0 K 2024-12-09T06:16:02,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/cf8dc57fa8f243a69a1c9297b041dd1e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/cf8dc57fa8f243a69a1c9297b041dd1e 2024-12-09T06:16:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/cf8dc57fa8f243a69a1c9297b041dd1e, entries=150, sequenceid=634, filesize=12.0 K 2024-12-09T06:16:02,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for 89f73a4239a4d6e12de44ad55db0a5e8 in 862ms, sequenceid=634, compaction requested=false 2024-12-09T06:16:02,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 89f73a4239a4d6e12de44ad55db0a5e8: 2024-12-09T06:16:05,018 INFO [Thread-146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-146. 2024-12-09T06:16:05,019 DEBUG [Thread-146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:05,019 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:05,019 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:05,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:05,078 INFO [Thread-150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-150. 2024-12-09T06:16:05,078 DEBUG [Thread-150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:05,078 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:05,078 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:05,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:05,108 INFO [Thread-144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-144. 2024-12-09T06:16:05,108 DEBUG [Thread-144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:05,108 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:05,109 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:05,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 189 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 219 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3366 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3319 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T06:16:05,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1536 2024-12-09T06:16:05,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4608 rows 2024-12-09T06:16:05,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1533 2024-12-09T06:16:05,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4599 rows 2024-12-09T06:16:05,110 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T06:16:05,110 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:16:05,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:05,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:05,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:05,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T06:16:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T06:16:05,124 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724965123"}]},"ts":"1733724965123"} 2024-12-09T06:16:05,126 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T06:16:05,126 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T06:16:05,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:16:05,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, UNASSIGN}] 2024-12-09T06:16:05,133 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, UNASSIGN 2024-12-09T06:16:05,135 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=89f73a4239a4d6e12de44ad55db0a5e8, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:05,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, UNASSIGN because future has completed 2024-12-09T06:16:05,138 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:16:05,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:16:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T06:16:05,296 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:16:05,296 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:16:05,297 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 89f73a4239a4d6e12de44ad55db0a5e8, disabling compactions & flushes 2024-12-09T06:16:05,297 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:16:05,297 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:16:05,297 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. after waiting 0 ms 2024-12-09T06:16:05,297 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:16:05,297 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 89f73a4239a4d6e12de44ad55db0a5e8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T06:16:05,297 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=A 2024-12-09T06:16:05,297 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:05,297 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=B 2024-12-09T06:16:05,297 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:05,298 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 89f73a4239a4d6e12de44ad55db0a5e8, store=C 2024-12-09T06:16:05,298 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:05,301 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/27f083e418af47db8706846ba08b38de is 50, key is test_row_1/A:col10/1733724965107/Put/seqid=0 2024-12-09T06:16:05,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741941_1117 (size=9857) 2024-12-09T06:16:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T06:16:05,705 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/27f083e418af47db8706846ba08b38de 2024-12-09T06:16:05,715 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/98e0bbc9502242c3b22f7cb95c0f6ff4 is 50, key is test_row_1/B:col10/1733724965107/Put/seqid=0 2024-12-09T06:16:05,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741942_1118 (size=9857) 2024-12-09T06:16:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T06:16:06,132 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/98e0bbc9502242c3b22f7cb95c0f6ff4 2024-12-09T06:16:06,141 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/94d281fdd9844528a600fdebd6d762b8 is 50, key is test_row_1/C:col10/1733724965107/Put/seqid=0 2024-12-09T06:16:06,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741943_1119 (size=9857) 2024-12-09T06:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T06:16:06,546 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/94d281fdd9844528a600fdebd6d762b8 2024-12-09T06:16:06,551 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/A/27f083e418af47db8706846ba08b38de as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/27f083e418af47db8706846ba08b38de 2024-12-09T06:16:06,557 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/27f083e418af47db8706846ba08b38de, entries=100, sequenceid=642, filesize=9.6 K 2024-12-09T06:16:06,558 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/B/98e0bbc9502242c3b22f7cb95c0f6ff4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/98e0bbc9502242c3b22f7cb95c0f6ff4 2024-12-09T06:16:06,562 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/98e0bbc9502242c3b22f7cb95c0f6ff4, entries=100, sequenceid=642, filesize=9.6 K 2024-12-09T06:16:06,563 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/.tmp/C/94d281fdd9844528a600fdebd6d762b8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/94d281fdd9844528a600fdebd6d762b8 2024-12-09T06:16:06,567 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/94d281fdd9844528a600fdebd6d762b8, entries=100, sequenceid=642, filesize=9.6 K 2024-12-09T06:16:06,568 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 89f73a4239a4d6e12de44ad55db0a5e8 in 1271ms, sequenceid=642, compaction requested=true 2024-12-09T06:16:06,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/aa44851b3fcb4567bded30e4c0754086, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0ea69aba66804923b80beb2b22c00221, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/763b9257a77a41e69eefd63824843712, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7629bb0f46f1494b921ffebdcc8944d0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d5ed581db8cc425fbd8cf551fc2fe1e3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/dd1a1fa1dc1b48938433a895d480610a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b29dc5dd204147f693fd553e7eebd79b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d33e03724a7a4b068aace3a54b120786, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/15a8a5f764e947f8adf30e5e18d9e287, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/af39531140a541e697ddd65f9fc8eb67, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0d794cb8a6164141a9c75b7330db2ad0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/41dba62976cf4a13b3ad58644cf3cf93, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/167dcf53775c471a8c6aeb2777a09915, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bfa8952c97ca4860afc14f58ebd2a6d7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b50f4212464348b097932310e027315b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5218bbd299814ab2bf930582eb279d68, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f14f55df06bd4faf908a959c3fce6e69, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/2dedcf085d3e424ba7734ecba2dd9671, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/47fd9923c33a4cf38404324ed79f7c1a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/c1e7cb9253a44c49b6986974227bd7ab, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5d9368d6b5c5447ca9c5a7fd82836bf9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/ffa0fe813006447c9810b94d328a02da, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/11da901592334ef89d4b665c165aa22a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a4719abb79e146b5b5ea41e15a0fa05c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bdb3f3a4968c46ff8350ab9813588720, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f28eff85d36d4538a6730057b3d36d57, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a3683e89a9574ab88f2d1b5c2495fbc4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/06b7035808694a26a263718a1f1b062b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/131dc694c4464007909db5f482ca8fd3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a7090bd3c83847cd9ba3f3fece7abb2f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/e0abece07f1f4b7998332ebc5168d1eb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/77651f70f1274d879cebc958158dadab] to archive 2024-12-09T06:16:06,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:06,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/aa44851b3fcb4567bded30e4c0754086 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/aa44851b3fcb4567bded30e4c0754086 2024-12-09T06:16:06,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0ea69aba66804923b80beb2b22c00221 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0ea69aba66804923b80beb2b22c00221 2024-12-09T06:16:06,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/763b9257a77a41e69eefd63824843712 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/763b9257a77a41e69eefd63824843712 2024-12-09T06:16:06,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7629bb0f46f1494b921ffebdcc8944d0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7629bb0f46f1494b921ffebdcc8944d0 2024-12-09T06:16:06,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d5ed581db8cc425fbd8cf551fc2fe1e3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d5ed581db8cc425fbd8cf551fc2fe1e3 2024-12-09T06:16:06,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/dd1a1fa1dc1b48938433a895d480610a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/dd1a1fa1dc1b48938433a895d480610a 2024-12-09T06:16:06,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b29dc5dd204147f693fd553e7eebd79b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b29dc5dd204147f693fd553e7eebd79b 2024-12-09T06:16:06,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d33e03724a7a4b068aace3a54b120786 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/d33e03724a7a4b068aace3a54b120786 2024-12-09T06:16:06,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/15a8a5f764e947f8adf30e5e18d9e287 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/15a8a5f764e947f8adf30e5e18d9e287 2024-12-09T06:16:06,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/af39531140a541e697ddd65f9fc8eb67 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/af39531140a541e697ddd65f9fc8eb67 2024-12-09T06:16:06,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0d794cb8a6164141a9c75b7330db2ad0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/0d794cb8a6164141a9c75b7330db2ad0 2024-12-09T06:16:06,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/41dba62976cf4a13b3ad58644cf3cf93 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/41dba62976cf4a13b3ad58644cf3cf93 2024-12-09T06:16:06,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/167dcf53775c471a8c6aeb2777a09915 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/167dcf53775c471a8c6aeb2777a09915 2024-12-09T06:16:06,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bfa8952c97ca4860afc14f58ebd2a6d7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bfa8952c97ca4860afc14f58ebd2a6d7 2024-12-09T06:16:06,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b50f4212464348b097932310e027315b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/b50f4212464348b097932310e027315b 2024-12-09T06:16:06,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5218bbd299814ab2bf930582eb279d68 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5218bbd299814ab2bf930582eb279d68 2024-12-09T06:16:06,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f14f55df06bd4faf908a959c3fce6e69 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f14f55df06bd4faf908a959c3fce6e69 2024-12-09T06:16:06,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/2dedcf085d3e424ba7734ecba2dd9671 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/2dedcf085d3e424ba7734ecba2dd9671 2024-12-09T06:16:06,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/47fd9923c33a4cf38404324ed79f7c1a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/47fd9923c33a4cf38404324ed79f7c1a 2024-12-09T06:16:06,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/c1e7cb9253a44c49b6986974227bd7ab to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/c1e7cb9253a44c49b6986974227bd7ab 2024-12-09T06:16:06,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5d9368d6b5c5447ca9c5a7fd82836bf9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5d9368d6b5c5447ca9c5a7fd82836bf9 2024-12-09T06:16:06,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/ffa0fe813006447c9810b94d328a02da to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/ffa0fe813006447c9810b94d328a02da 2024-12-09T06:16:06,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/11da901592334ef89d4b665c165aa22a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/11da901592334ef89d4b665c165aa22a 2024-12-09T06:16:06,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a4719abb79e146b5b5ea41e15a0fa05c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a4719abb79e146b5b5ea41e15a0fa05c 2024-12-09T06:16:06,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bdb3f3a4968c46ff8350ab9813588720 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/bdb3f3a4968c46ff8350ab9813588720 2024-12-09T06:16:06,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f28eff85d36d4538a6730057b3d36d57 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/f28eff85d36d4538a6730057b3d36d57 2024-12-09T06:16:06,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/7c6ef5e9a9cd40ef9f2ce9f54a7e8f3f 2024-12-09T06:16:06,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a3683e89a9574ab88f2d1b5c2495fbc4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a3683e89a9574ab88f2d1b5c2495fbc4 2024-12-09T06:16:06,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/06b7035808694a26a263718a1f1b062b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/06b7035808694a26a263718a1f1b062b 2024-12-09T06:16:06,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/131dc694c4464007909db5f482ca8fd3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/131dc694c4464007909db5f482ca8fd3 2024-12-09T06:16:06,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a7090bd3c83847cd9ba3f3fece7abb2f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a7090bd3c83847cd9ba3f3fece7abb2f 2024-12-09T06:16:06,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/e0abece07f1f4b7998332ebc5168d1eb to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/e0abece07f1f4b7998332ebc5168d1eb 2024-12-09T06:16:06,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/77651f70f1274d879cebc958158dadab to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/77651f70f1274d879cebc958158dadab 2024-12-09T06:16:06,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/7e82557b40104e538de666e2e04e6ad0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a8407467cb284a409d07c790479085b6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/753b5e3ce5de401ebc0bca82ae589cf6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/50ed96a0062d4778b7865827b9c8c8c8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3295a50f5f0b4a9ca0d070fbb1f84013, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/0923c64ed0d44a9f8b68a8b55f931096, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b03ce88d8e204a50b26f3cbfc1cc6853, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/bc5ae7bf1cc14626a0c6586269199902, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a7669c9c013b4692ad388de28b2987c1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/2873d493714a458e9862814b16300fba, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ffdb9d22cf5f4936b4262df26988809a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/f0479acd98f849009d2597c1be224efb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/509ed62ca73a438182574a53cdcd51c3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/cb8fc737db2845989010eb56cd9c132c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/68bf1ac7f731425ca52ff17a06b0477e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/071e2c67978e4d33ae5a4d036c1f5684, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a221b4774269407aab459a383c63c35b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5740c30009dc41c9ad38a643660f50f8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/64dc10953972409b87715b63ae6fb9f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a5b593f12380498cbfabad2bb0ac15fb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/34aa8c808fd44653bffc9c64f7e09481, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b4ce7b767eed42c68b091592973f2a91, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/182ce5e7fba54da5953b7c43e76a5176, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9b5098e07c74454c97437f913302c426, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3fe2901a8bba417f9920c75c4b07ca51, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/67e137bd64404a8b909410df7da5a95b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/d8d64e1155224be9b50664585ab4bd38, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/86423159560a448a908b00cd2f74213c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5767ecde20b741b5832999d1001e62d6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ef8aac5070474396bfb4a3a02ef6ec5d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/fd6cc17353644b4d83e40a2c30037895, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9caa2efd7bb0449c8d26f8c0005efd3b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/8e7a0cda90f144aa8e19bba3a552265b] to archive 2024-12-09T06:16:06,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:06,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/7e82557b40104e538de666e2e04e6ad0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/7e82557b40104e538de666e2e04e6ad0 2024-12-09T06:16:06,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a8407467cb284a409d07c790479085b6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a8407467cb284a409d07c790479085b6 2024-12-09T06:16:06,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/753b5e3ce5de401ebc0bca82ae589cf6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/753b5e3ce5de401ebc0bca82ae589cf6 2024-12-09T06:16:06,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/50ed96a0062d4778b7865827b9c8c8c8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/50ed96a0062d4778b7865827b9c8c8c8 2024-12-09T06:16:06,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3295a50f5f0b4a9ca0d070fbb1f84013 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3295a50f5f0b4a9ca0d070fbb1f84013 2024-12-09T06:16:06,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/0923c64ed0d44a9f8b68a8b55f931096 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/0923c64ed0d44a9f8b68a8b55f931096 2024-12-09T06:16:06,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b03ce88d8e204a50b26f3cbfc1cc6853 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b03ce88d8e204a50b26f3cbfc1cc6853 2024-12-09T06:16:06,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/bc5ae7bf1cc14626a0c6586269199902 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/bc5ae7bf1cc14626a0c6586269199902 2024-12-09T06:16:06,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a7669c9c013b4692ad388de28b2987c1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a7669c9c013b4692ad388de28b2987c1 2024-12-09T06:16:06,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/2873d493714a458e9862814b16300fba to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/2873d493714a458e9862814b16300fba 2024-12-09T06:16:06,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ffdb9d22cf5f4936b4262df26988809a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ffdb9d22cf5f4936b4262df26988809a 2024-12-09T06:16:06,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/f0479acd98f849009d2597c1be224efb to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/f0479acd98f849009d2597c1be224efb 2024-12-09T06:16:06,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/509ed62ca73a438182574a53cdcd51c3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/509ed62ca73a438182574a53cdcd51c3 2024-12-09T06:16:06,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/cb8fc737db2845989010eb56cd9c132c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/cb8fc737db2845989010eb56cd9c132c 2024-12-09T06:16:06,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/68bf1ac7f731425ca52ff17a06b0477e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/68bf1ac7f731425ca52ff17a06b0477e 2024-12-09T06:16:06,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/071e2c67978e4d33ae5a4d036c1f5684 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/071e2c67978e4d33ae5a4d036c1f5684 2024-12-09T06:16:06,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a221b4774269407aab459a383c63c35b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a221b4774269407aab459a383c63c35b 2024-12-09T06:16:06,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5740c30009dc41c9ad38a643660f50f8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5740c30009dc41c9ad38a643660f50f8 2024-12-09T06:16:06,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/64dc10953972409b87715b63ae6fb9f3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/64dc10953972409b87715b63ae6fb9f3 2024-12-09T06:16:06,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a5b593f12380498cbfabad2bb0ac15fb to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/a5b593f12380498cbfabad2bb0ac15fb 2024-12-09T06:16:06,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/34aa8c808fd44653bffc9c64f7e09481 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/34aa8c808fd44653bffc9c64f7e09481 2024-12-09T06:16:06,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b4ce7b767eed42c68b091592973f2a91 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/b4ce7b767eed42c68b091592973f2a91 2024-12-09T06:16:06,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/182ce5e7fba54da5953b7c43e76a5176 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/182ce5e7fba54da5953b7c43e76a5176 2024-12-09T06:16:06,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9b5098e07c74454c97437f913302c426 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9b5098e07c74454c97437f913302c426 2024-12-09T06:16:06,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3fe2901a8bba417f9920c75c4b07ca51 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3fe2901a8bba417f9920c75c4b07ca51 2024-12-09T06:16:06,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/67e137bd64404a8b909410df7da5a95b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/67e137bd64404a8b909410df7da5a95b 2024-12-09T06:16:06,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/d8d64e1155224be9b50664585ab4bd38 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/d8d64e1155224be9b50664585ab4bd38 2024-12-09T06:16:06,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/86423159560a448a908b00cd2f74213c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/86423159560a448a908b00cd2f74213c 2024-12-09T06:16:06,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5767ecde20b741b5832999d1001e62d6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/5767ecde20b741b5832999d1001e62d6 2024-12-09T06:16:06,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ef8aac5070474396bfb4a3a02ef6ec5d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/ef8aac5070474396bfb4a3a02ef6ec5d 2024-12-09T06:16:06,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/fd6cc17353644b4d83e40a2c30037895 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/fd6cc17353644b4d83e40a2c30037895 2024-12-09T06:16:06,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9caa2efd7bb0449c8d26f8c0005efd3b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/9caa2efd7bb0449c8d26f8c0005efd3b 2024-12-09T06:16:06,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/8e7a0cda90f144aa8e19bba3a552265b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/8e7a0cda90f144aa8e19bba3a552265b 2024-12-09T06:16:06,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/92bb28539f8a43ce9f87976254bda142, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/752035daae394490808b33b382c82d10, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/d373bf3d80b64868a7904049e24ee415, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e13b6c4778ba4abf85f5f74d0229edbe, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/daff047328fc4beaaedf299ea9cfd6dd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/0bb392ad45d84281921cefb21b785ad3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e93bdd2e03134e6aae7e920f4b214fa9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/1ae2315e8206434abdcaeb158762d500, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/beacae8e0d4a48d7b3dc4d8459c18b99, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/2d7b4a8fbaac425b88b999430ea88da1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/744b509a8d8940b08afaecfc27d9962a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b9d157b16a0f4c4d95ab7bc678d3ac11, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/5d3afd0c182844778518e5f4fbc2d09a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/4182bdfabe13404db6446c95a91fb21b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/9ec976c4961146ae897d767ad0d9c05a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/bc828cc6f6f340abb6cb7e805597b4aa, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/86da9095f4144624ac304ea1eef8919e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/527ccfb66acf40a59de49f6ca599b481, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/473e5e617d6e419d849544c945942e27, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/606ef2b1c73543f2992a85d0ccb69f2d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/be1becb08d5f484995ceabf43bc0743f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/63e0f95edb144787a8f138fc9c1a20cd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6cb5e9b4c95d492190c7e849cb6597c7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e06aba88d0d84b9ca20482eb3f290d4c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a9b510d6fac34529b7ac9f942bd96a36, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/37ecec39268e4ab085634e94274563a1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/96f3f3678a82471c958bf4c368935afe, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b1ab2bd80fde41c1902f29efff0761aa, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/7b1a3e86cd1b460d90dedffb7ba78d87, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/444c43b20f1540b0bd8ecdbc33fb28a2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a13c60ea147b42e99be12215160727a2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/58607def505e4e51bf6226cd011fa891] to archive 2024-12-09T06:16:06,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:06,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/92bb28539f8a43ce9f87976254bda142 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/92bb28539f8a43ce9f87976254bda142 2024-12-09T06:16:06,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/752035daae394490808b33b382c82d10 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/752035daae394490808b33b382c82d10 2024-12-09T06:16:06,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/d373bf3d80b64868a7904049e24ee415 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/d373bf3d80b64868a7904049e24ee415 2024-12-09T06:16:06,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e13b6c4778ba4abf85f5f74d0229edbe to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e13b6c4778ba4abf85f5f74d0229edbe 2024-12-09T06:16:06,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/daff047328fc4beaaedf299ea9cfd6dd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/daff047328fc4beaaedf299ea9cfd6dd 2024-12-09T06:16:06,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/0bb392ad45d84281921cefb21b785ad3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/0bb392ad45d84281921cefb21b785ad3 2024-12-09T06:16:06,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e93bdd2e03134e6aae7e920f4b214fa9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e93bdd2e03134e6aae7e920f4b214fa9 2024-12-09T06:16:06,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/1ae2315e8206434abdcaeb158762d500 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/1ae2315e8206434abdcaeb158762d500 2024-12-09T06:16:06,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/beacae8e0d4a48d7b3dc4d8459c18b99 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/beacae8e0d4a48d7b3dc4d8459c18b99 2024-12-09T06:16:06,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/2d7b4a8fbaac425b88b999430ea88da1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/2d7b4a8fbaac425b88b999430ea88da1 2024-12-09T06:16:06,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/744b509a8d8940b08afaecfc27d9962a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/744b509a8d8940b08afaecfc27d9962a 2024-12-09T06:16:06,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b9d157b16a0f4c4d95ab7bc678d3ac11 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b9d157b16a0f4c4d95ab7bc678d3ac11 2024-12-09T06:16:06,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/5d3afd0c182844778518e5f4fbc2d09a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/5d3afd0c182844778518e5f4fbc2d09a 2024-12-09T06:16:06,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/4182bdfabe13404db6446c95a91fb21b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/4182bdfabe13404db6446c95a91fb21b 2024-12-09T06:16:06,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/9ec976c4961146ae897d767ad0d9c05a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/9ec976c4961146ae897d767ad0d9c05a 2024-12-09T06:16:06,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/bc828cc6f6f340abb6cb7e805597b4aa to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/bc828cc6f6f340abb6cb7e805597b4aa 2024-12-09T06:16:06,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/86da9095f4144624ac304ea1eef8919e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/86da9095f4144624ac304ea1eef8919e 2024-12-09T06:16:06,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/527ccfb66acf40a59de49f6ca599b481 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/527ccfb66acf40a59de49f6ca599b481 2024-12-09T06:16:06,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/473e5e617d6e419d849544c945942e27 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/473e5e617d6e419d849544c945942e27 2024-12-09T06:16:06,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/606ef2b1c73543f2992a85d0ccb69f2d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/606ef2b1c73543f2992a85d0ccb69f2d 2024-12-09T06:16:06,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/be1becb08d5f484995ceabf43bc0743f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/be1becb08d5f484995ceabf43bc0743f 2024-12-09T06:16:06,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/63e0f95edb144787a8f138fc9c1a20cd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/63e0f95edb144787a8f138fc9c1a20cd 2024-12-09T06:16:06,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6cb5e9b4c95d492190c7e849cb6597c7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6cb5e9b4c95d492190c7e849cb6597c7 2024-12-09T06:16:06,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e06aba88d0d84b9ca20482eb3f290d4c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/e06aba88d0d84b9ca20482eb3f290d4c 2024-12-09T06:16:06,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a9b510d6fac34529b7ac9f942bd96a36 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a9b510d6fac34529b7ac9f942bd96a36 2024-12-09T06:16:06,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/37ecec39268e4ab085634e94274563a1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/37ecec39268e4ab085634e94274563a1 2024-12-09T06:16:06,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/96f3f3678a82471c958bf4c368935afe to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/96f3f3678a82471c958bf4c368935afe 2024-12-09T06:16:06,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b1ab2bd80fde41c1902f29efff0761aa to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/b1ab2bd80fde41c1902f29efff0761aa 2024-12-09T06:16:06,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/7b1a3e86cd1b460d90dedffb7ba78d87 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/7b1a3e86cd1b460d90dedffb7ba78d87 2024-12-09T06:16:06,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/444c43b20f1540b0bd8ecdbc33fb28a2 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/444c43b20f1540b0bd8ecdbc33fb28a2 2024-12-09T06:16:06,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a13c60ea147b42e99be12215160727a2 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/a13c60ea147b42e99be12215160727a2 2024-12-09T06:16:06,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/58607def505e4e51bf6226cd011fa891 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/58607def505e4e51bf6226cd011fa891 2024-12-09T06:16:06,730 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/recovered.edits/645.seqid, newMaxSeqId=645, maxSeqId=1 2024-12-09T06:16:06,732 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. 2024-12-09T06:16:06,732 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 89f73a4239a4d6e12de44ad55db0a5e8: Waiting for close lock at 1733724965297Running coprocessor pre-close hooks at 1733724965297Disabling compacts and flushes for region at 1733724965297Disabling writes for close at 1733724965297Obtaining lock to block concurrent updates at 1733724965297Preparing flush snapshotting stores in 89f73a4239a4d6e12de44ad55db0a5e8 at 1733724965297Finished memstore snapshotting TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733724965298 (+1 ms)Flushing stores of TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8. at 1733724965298Flushing 89f73a4239a4d6e12de44ad55db0a5e8/A: creating writer at 1733724965298Flushing 89f73a4239a4d6e12de44ad55db0a5e8/A: appending metadata at 1733724965301 (+3 ms)Flushing 89f73a4239a4d6e12de44ad55db0a5e8/A: closing flushed file at 1733724965301Flushing 89f73a4239a4d6e12de44ad55db0a5e8/B: creating writer at 1733724965711 (+410 ms)Flushing 89f73a4239a4d6e12de44ad55db0a5e8/B: appending metadata at 1733724965715 (+4 ms)Flushing 89f73a4239a4d6e12de44ad55db0a5e8/B: closing flushed file at 1733724965715Flushing 89f73a4239a4d6e12de44ad55db0a5e8/C: creating writer at 1733724966137 (+422 ms)Flushing 89f73a4239a4d6e12de44ad55db0a5e8/C: appending metadata at 1733724966140 (+3 ms)Flushing 89f73a4239a4d6e12de44ad55db0a5e8/C: closing flushed file at 1733724966140Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b07d836: reopening flushed file at 1733724966551 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e6d2b11: reopening flushed file at 1733724966557 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70486cf9: reopening flushed file at 1733724966562 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 89f73a4239a4d6e12de44ad55db0a5e8 in 1271ms, sequenceid=642, compaction requested=true at 1733724966568 (+6 ms)Writing region close event to WAL at 1733724966727 (+159 ms)Running coprocessor post-close hooks at 1733724966731 (+4 ms)Closed at 1733724966732 (+1 ms) 2024-12-09T06:16:06,735 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:16:06,735 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=89f73a4239a4d6e12de44ad55db0a5e8, regionState=CLOSED 2024-12-09T06:16:06,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:16:06,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T06:16:06,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 89f73a4239a4d6e12de44ad55db0a5e8, server=82ac79f0323b,42797,1733724930590 in 1.6010 sec 2024-12-09T06:16:06,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-09T06:16:06,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=89f73a4239a4d6e12de44ad55db0a5e8, UNASSIGN in 1.6100 sec 2024-12-09T06:16:06,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T06:16:06,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6170 sec 2024-12-09T06:16:06,747 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724966747"}]},"ts":"1733724966747"} 2024-12-09T06:16:06,749 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T06:16:06,749 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T06:16:06,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6340 sec 2024-12-09T06:16:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T06:16:07,267 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:07,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T06:16:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:07,274 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T06:16:07,276 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:07,282 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:16:07,290 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/recovered.edits] 2024-12-09T06:16:07,296 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/27f083e418af47db8706846ba08b38de to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/27f083e418af47db8706846ba08b38de 2024-12-09T06:16:07,297 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5920da35d7c541109c6b0d7eb44cefcb to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/5920da35d7c541109c6b0d7eb44cefcb 2024-12-09T06:16:07,298 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a78b2f28dd1041efb07928c88ddfc29d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/A/a78b2f28dd1041efb07928c88ddfc29d 2024-12-09T06:16:07,301 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/1b6a36741b2247359e7e97d095aa6e54 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/1b6a36741b2247359e7e97d095aa6e54 2024-12-09T06:16:07,302 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3369d5380cd74808a9a17a096877572d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/3369d5380cd74808a9a17a096877572d 2024-12-09T06:16:07,303 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/98e0bbc9502242c3b22f7cb95c0f6ff4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/B/98e0bbc9502242c3b22f7cb95c0f6ff4 2024-12-09T06:16:07,306 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6b2b31e61e8843769a1386b2e350fcb8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/6b2b31e61e8843769a1386b2e350fcb8 2024-12-09T06:16:07,307 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/94d281fdd9844528a600fdebd6d762b8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/94d281fdd9844528a600fdebd6d762b8 2024-12-09T06:16:07,309 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/cf8dc57fa8f243a69a1c9297b041dd1e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/C/cf8dc57fa8f243a69a1c9297b041dd1e 2024-12-09T06:16:07,312 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/recovered.edits/645.seqid to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8/recovered.edits/645.seqid 2024-12-09T06:16:07,313 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/89f73a4239a4d6e12de44ad55db0a5e8 2024-12-09T06:16:07,313 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T06:16:07,319 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:07,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-09T06:16:07,333 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T06:16:07,373 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T06:16:07,374 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:07,374 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T06:16:07,375 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733724967374"}]},"ts":"9223372036854775807"} 2024-12-09T06:16:07,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T06:16:07,382 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 89f73a4239a4d6e12de44ad55db0a5e8, NAME => 'TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T06:16:07,382 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T06:16:07,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733724967382"}]},"ts":"9223372036854775807"} 2024-12-09T06:16:07,385 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T06:16:07,386 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T06:16:07,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 116 msec 2024-12-09T06:16:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T06:16:07,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T06:16:07,597 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:07,608 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=245 (was 209) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;82ac79f0323b:42797-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_45612531_22 at /127.0.0.1:44308 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=993 (was 902) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3872 (was 2453) - AvailableMemoryMB LEAK? - 2024-12-09T06:16:07,618 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=245, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=993, ProcessCount=11, AvailableMemoryMB=3872 2024-12-09T06:16:07,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:16:07,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:16:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:07,622 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T06:16:07,622 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:07,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-09T06:16:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T06:16:07,623 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T06:16:07,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741944_1120 (size=1061) 2024-12-09T06:16:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T06:16:07,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T06:16:08,031 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d 2024-12-09T06:16:08,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741945_1121 (size=53) 2024-12-09T06:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T06:16:08,449 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:16:08,449 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing edaa2b3dd1776fd6538daec178f1f68f, disabling compactions & flushes 2024-12-09T06:16:08,449 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:08,450 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:08,450 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. after waiting 0 ms 2024-12-09T06:16:08,450 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:08,450 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:08,450 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for edaa2b3dd1776fd6538daec178f1f68f: Waiting for close lock at 1733724968449Disabling compacts and flushes for region at 1733724968449Disabling writes for close at 1733724968450 (+1 ms)Writing region close event to WAL at 1733724968450Closed at 1733724968450 2024-12-09T06:16:08,451 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T06:16:08,451 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733724968451"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733724968451"}]},"ts":"1733724968451"} 2024-12-09T06:16:08,453 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T06:16:08,454 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T06:16:08,454 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724968454"}]},"ts":"1733724968454"} 2024-12-09T06:16:08,457 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T06:16:08,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, ASSIGN}] 2024-12-09T06:16:08,459 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, ASSIGN 2024-12-09T06:16:08,459 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, ASSIGN; state=OFFLINE, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=false 2024-12-09T06:16:08,610 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=edaa2b3dd1776fd6538daec178f1f68f, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:08,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, ASSIGN because future has completed 2024-12-09T06:16:08,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:16:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T06:16:08,786 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:08,786 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:16:08,786 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,786 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:16:08,787 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,787 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,797 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,805 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:08,807 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaa2b3dd1776fd6538daec178f1f68f columnFamilyName A 2024-12-09T06:16:08,807 DEBUG [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:08,807 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(327): Store=edaa2b3dd1776fd6538daec178f1f68f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:08,808 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,814 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:08,814 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaa2b3dd1776fd6538daec178f1f68f columnFamilyName B 2024-12-09T06:16:08,814 DEBUG [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:08,817 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(327): Store=edaa2b3dd1776fd6538daec178f1f68f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:08,817 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,821 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:08,821 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaa2b3dd1776fd6538daec178f1f68f columnFamilyName C 2024-12-09T06:16:08,821 DEBUG [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:08,830 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(327): Store=edaa2b3dd1776fd6538daec178f1f68f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:08,830 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:08,830 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,833 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,838 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,847 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,847 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,852 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:16:08,862 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,877 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:16:08,877 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened edaa2b3dd1776fd6538daec178f1f68f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68461645, jitterRate=0.020158007740974426}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:16:08,878 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:08,879 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for edaa2b3dd1776fd6538daec178f1f68f: Running coprocessor pre-open hook at 1733724968787Writing region info on filesystem at 1733724968787Initializing all the Stores at 1733724968793 (+6 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724968793Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724968794 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724968794Cleaning up temporary data from old regions at 1733724968847 (+53 ms)Running coprocessor post-open hooks at 1733724968878 (+31 ms)Region opened successfully at 1733724968879 (+1 ms) 2024-12-09T06:16:08,892 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., pid=36, masterSystemTime=1733724968775 2024-12-09T06:16:08,894 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:08,894 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:08,895 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=edaa2b3dd1776fd6538daec178f1f68f, regionState=OPEN, openSeqNum=2, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:08,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:16:08,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-09T06:16:08,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 in 287 msec 2024-12-09T06:16:08,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-09T06:16:08,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, ASSIGN in 446 msec 2024-12-09T06:16:08,908 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T06:16:08,908 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724968908"}]},"ts":"1733724968908"} 2024-12-09T06:16:08,921 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T06:16:08,923 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T06:16:08,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3030 sec 2024-12-09T06:16:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T06:16:09,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:09,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:09,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:09,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:09,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526766c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:09,781 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:09,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4f572f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:09,782 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:09,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:09,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:09,786 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T06:16:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 82ac79f0323b,43241,1733724929416 2024-12-09T06:16:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@317c75cf 2024-12-09T06:16:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T06:16:09,788 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T06:16:09,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:16:09,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:16:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:09,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741946_1122 (size=1097) 2024-12-09T06:16:10,221 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T06:16:10,221 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T06:16:10,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:16:10,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, REOPEN/MOVE}] 2024-12-09T06:16:10,235 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, REOPEN/MOVE 2024-12-09T06:16:10,236 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=edaa2b3dd1776fd6538daec178f1f68f, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:10,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, REOPEN/MOVE because future has completed 2024-12-09T06:16:10,238 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:16:10,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:16:10,390 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,390 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:16:10,390 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing edaa2b3dd1776fd6538daec178f1f68f, disabling compactions & flushes 2024-12-09T06:16:10,391 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:10,391 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:10,391 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. after waiting 0 ms 2024-12-09T06:16:10,391 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:10,394 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T06:16:10,395 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:10,395 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for edaa2b3dd1776fd6538daec178f1f68f: Waiting for close lock at 1733724970390Running coprocessor pre-close hooks at 1733724970390Disabling compacts and flushes for region at 1733724970390Disabling writes for close at 1733724970391 (+1 ms)Writing region close event to WAL at 1733724970391Running coprocessor post-close hooks at 1733724970395 (+4 ms)Closed at 1733724970395 2024-12-09T06:16:10,395 WARN [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: edaa2b3dd1776fd6538daec178f1f68f to self. 2024-12-09T06:16:10,397 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,398 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=edaa2b3dd1776fd6538daec178f1f68f, regionState=CLOSED 2024-12-09T06:16:10,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:16:10,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-09T06:16:10,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 in 162 msec 2024-12-09T06:16:10,403 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, REOPEN/MOVE; state=CLOSED, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=true 2024-12-09T06:16:10,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=edaa2b3dd1776fd6538daec178f1f68f, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:10,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, REOPEN/MOVE because future has completed 2024-12-09T06:16:10,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:16:10,713 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:10,714 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:16:10,714 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,714 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:16:10,714 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,714 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,719 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,720 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:10,733 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaa2b3dd1776fd6538daec178f1f68f columnFamilyName A 2024-12-09T06:16:10,757 DEBUG [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:10,757 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(327): Store=edaa2b3dd1776fd6538daec178f1f68f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:10,758 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,762 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:10,762 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaa2b3dd1776fd6538daec178f1f68f columnFamilyName B 2024-12-09T06:16:10,762 DEBUG [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:10,765 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(327): Store=edaa2b3dd1776fd6538daec178f1f68f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:10,766 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,767 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:10,767 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edaa2b3dd1776fd6538daec178f1f68f columnFamilyName C 2024-12-09T06:16:10,767 DEBUG [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:10,768 INFO [StoreOpener-edaa2b3dd1776fd6538daec178f1f68f-1 {}] regionserver.HStore(327): Store=edaa2b3dd1776fd6538daec178f1f68f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:10,768 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:10,768 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,769 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,771 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,772 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,772 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,772 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:16:10,775 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,776 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened edaa2b3dd1776fd6538daec178f1f68f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63812736, jitterRate=-0.04911613464355469}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:16:10,776 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,779 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for edaa2b3dd1776fd6538daec178f1f68f: Running coprocessor pre-open hook at 1733724970714Writing region info on filesystem at 1733724970714Initializing all the Stores at 1733724970717 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724970717Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724970718 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724970718Cleaning up temporary data from old regions at 1733724970772 (+54 ms)Running coprocessor post-open hooks at 1733724970776 (+4 ms)Region opened successfully at 1733724970779 (+3 ms) 2024-12-09T06:16:10,781 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., pid=41, masterSystemTime=1733724970708 2024-12-09T06:16:10,786 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:10,787 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:10,787 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=edaa2b3dd1776fd6538daec178f1f68f, regionState=OPEN, openSeqNum=5, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:10,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:16:10,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-09T06:16:10,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 in 236 msec 2024-12-09T06:16:10,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-09T06:16:10,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, REOPEN/MOVE in 559 msec 2024-12-09T06:16:10,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-09T06:16:10,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 573 msec 2024-12-09T06:16:10,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0070 sec 2024-12-09T06:16:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-09T06:16:10,811 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:10,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,814 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a627c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,816 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c039f30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,822 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b3e649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,825 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242601ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,833 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@187b8281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e30fc0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,838 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a8a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,839 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15149708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,843 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ff477b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,844 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314270ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,848 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc5d73b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42264124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,853 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bba3eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,855 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da09975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,859 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33518209, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,861 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d98bf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:10,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:10,864 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0b8635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:10,867 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:10,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ced968b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:10,875 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,875 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,877 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:10,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,880 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-09T06:16:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,889 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,892 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T06:16:10,895 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,907 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:10,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:10,921 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:10,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:10,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:10,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:10,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:10,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:10,930 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,933 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:10,934 DEBUG [Thread-635 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:10,934 DEBUG [Thread-635 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:16:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T06:16:11,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724970992, max timestamp=1733724971028Num uniques 859; 2024-12-09T06:16:11,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:11,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724971018, max timestamp=1733724971028Num uniques 859; 2024-12-09T06:16:11,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:11,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724971018, max timestamp=1733724971036Num uniques 859; 2024-12-09T06:16:11,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:11,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209888e5778aa8b41feaddaf01a828d3c6d_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724970894/Put/seqid=0 2024-12-09T06:16:11,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725031079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725031083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725031086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725031090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725031090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741947_1123 (size=12154) 2024-12-09T06:16:11,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:11,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725031197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725031197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T06:16:11,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:11,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725031207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725031207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725031209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:11,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:11,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725031406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725031417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725031417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725031419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725031426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T06:16:11,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:11,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:11,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209888e5778aa8b41feaddaf01a828d3c6d_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209888e5778aa8b41feaddaf01a828d3c6d_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:11,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/07889b634d7847d29a9a2ce26dd32c2d, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:11,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/07889b634d7847d29a9a2ce26dd32c2d is 175, key is test_row_0/A:col10/1733724970894/Put/seqid=0 2024-12-09T06:16:11,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741948_1124 (size=30955) 2024-12-09T06:16:11,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:11,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725031727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725031727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725031737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725031739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725031741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:11,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:11,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:11,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:11,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:11,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:12,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:12,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,024 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/07889b634d7847d29a9a2ce26dd32c2d 2024-12-09T06:16:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T06:16:12,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/4b6cffbc256547418b8be2f25589cfd9 is 50, key is test_row_0/B:col10/1733724970894/Put/seqid=0 2024-12-09T06:16:12,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741949_1125 (size=12001) 2024-12-09T06:16:12,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:12,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:12,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725032240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:12,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725032249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:12,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725032257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:12,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725032257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:12,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725032257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:12,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:12,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:12,288 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T06:16:12,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:12,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:12,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:12,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/4b6cffbc256547418b8be2f25589cfd9 2024-12-09T06:16:12,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c8623b538b58472491f0084e801d48be is 50, key is test_row_0/C:col10/1733724970894/Put/seqid=0 2024-12-09T06:16:12,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:12,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:12,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741950_1126 (size=12001) 2024-12-09T06:16:12,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:12,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:12,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:12,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:12,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:12,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c8623b538b58472491f0084e801d48be 2024-12-09T06:16:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T06:16:13,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/07889b634d7847d29a9a2ce26dd32c2d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/07889b634d7847d29a9a2ce26dd32c2d 2024-12-09T06:16:13,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/07889b634d7847d29a9a2ce26dd32c2d, entries=150, sequenceid=15, filesize=30.2 K 2024-12-09T06:16:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/4b6cffbc256547418b8be2f25589cfd9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/4b6cffbc256547418b8be2f25589cfd9 2024-12-09T06:16:13,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/4b6cffbc256547418b8be2f25589cfd9, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T06:16:13,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c8623b538b58472491f0084e801d48be as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c8623b538b58472491f0084e801d48be 2024-12-09T06:16:13,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c8623b538b58472491f0084e801d48be, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T06:16:13,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for edaa2b3dd1776fd6538daec178f1f68f in 2296ms, sequenceid=15, compaction requested=false 2024-12-09T06:16:13,217 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T06:16:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:13,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=143.13 KB heapSize=405.30 KB 2024-12-09T06:16:13,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:13,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:13,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:13,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:13,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:13,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:13,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:13,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:13,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209cff884ccc6a84c279a1c75329572972f_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724971083/Put/seqid=0 2024-12-09T06:16:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725033337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725033341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725033346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741951_1127 (size=14594) 2024-12-09T06:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725033351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725033360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725033458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725033468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725033468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:13,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725033468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725033477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725033666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725033677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725033681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725033686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725033696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:13,774 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209cff884ccc6a84c279a1c75329572972f_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cff884ccc6a84c279a1c75329572972f_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:13,784 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/036ef63581fa4037862b828b487e5538, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:13,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/036ef63581fa4037862b828b487e5538 is 175, key is test_row_0/A:col10/1733724971083/Put/seqid=0 2024-12-09T06:16:13,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:13,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:13,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741952_1128 (size=39549) 2024-12-09T06:16:13,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:13,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:13,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:13,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:13,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725033977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:13,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725033989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:13,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:13,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725033996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725033996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725034007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/036ef63581fa4037862b828b487e5538 2024-12-09T06:16:14,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/cf497e431e574e7987ac7349da8845c9 is 50, key is test_row_0/B:col10/1733724971083/Put/seqid=0 2024-12-09T06:16:14,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741953_1129 (size=12001) 2024-12-09T06:16:14,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/cf497e431e574e7987ac7349da8845c9 2024-12-09T06:16:14,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/ea907b96c6b546959c9f61203a7e1070 is 50, key is test_row_0/C:col10/1733724971083/Put/seqid=0 2024-12-09T06:16:14,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741954_1130 (size=12001) 2024-12-09T06:16:14,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/ea907b96c6b546959c9f61203a7e1070 2024-12-09T06:16:14,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/036ef63581fa4037862b828b487e5538 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/036ef63581fa4037862b828b487e5538 2024-12-09T06:16:14,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/036ef63581fa4037862b828b487e5538, entries=200, sequenceid=52, filesize=38.6 K 2024-12-09T06:16:14,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/cf497e431e574e7987ac7349da8845c9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/cf497e431e574e7987ac7349da8845c9 2024-12-09T06:16:14,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/cf497e431e574e7987ac7349da8845c9, entries=150, sequenceid=52, filesize=11.7 K 2024-12-09T06:16:14,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/ea907b96c6b546959c9f61203a7e1070 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/ea907b96c6b546959c9f61203a7e1070 2024-12-09T06:16:14,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/ea907b96c6b546959c9f61203a7e1070, entries=150, sequenceid=52, filesize=11.7 K 2024-12-09T06:16:14,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~405.25 KB/414976, currentSize=53.67 KB/54960 for edaa2b3dd1776fd6538daec178f1f68f in 1244ms, sequenceid=52, compaction requested=false 2024-12-09T06:16:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:14,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T06:16:14,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:14,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:14,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:14,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:14,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:14,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:14,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092c33b7dd15624c14884611880961e6d3_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724974519/Put/seqid=0 2024-12-09T06:16:14,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:14,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:14,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724974567, max timestamp=1733724974580Num uniques 859; 2024-12-09T06:16:14,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725034578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724974567, max timestamp=1733724974580Num uniques 859; 2024-12-09T06:16:14,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741955_1131 (size=17034) 2024-12-09T06:16:14,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724974567, max timestamp=1733724974580Num uniques 859; 2024-12-09T06:16:14,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:14,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:14,621 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092c33b7dd15624c14884611880961e6d3_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092c33b7dd15624c14884611880961e6d3_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:14,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/7486ef0d76774174a43aa2f3becc9ce2, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:14,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/7486ef0d76774174a43aa2f3becc9ce2 is 175, key is test_row_0/A:col10/1733724974519/Put/seqid=0 2024-12-09T06:16:14,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741956_1132 (size=48139) 2024-12-09T06:16:14,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/7486ef0d76774174a43aa2f3becc9ce2 2024-12-09T06:16:14,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725034645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725034651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725034666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725034667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/c0c8bce224e34152a79cabd79bf27c04 is 50, key is test_row_0/B:col10/1733724974519/Put/seqid=0 2024-12-09T06:16:14,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725034704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741957_1133 (size=12001) 2024-12-09T06:16:14,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:14,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:14,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/c0c8bce224e34152a79cabd79bf27c04 2024-12-09T06:16:14,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c49fc2c2cb6f4d7d8b42133174c92b6b is 50, key is test_row_0/C:col10/1733724974519/Put/seqid=0 2024-12-09T06:16:14,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725034778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725034783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725034787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725034787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741958_1134 (size=12001) 2024-12-09T06:16:14,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c49fc2c2cb6f4d7d8b42133174c92b6b 2024-12-09T06:16:14,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/7486ef0d76774174a43aa2f3becc9ce2 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/7486ef0d76774174a43aa2f3becc9ce2 2024-12-09T06:16:14,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:14,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:14,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:14,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725034917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:14,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/7486ef0d76774174a43aa2f3becc9ce2, entries=250, sequenceid=64, filesize=47.0 K 2024-12-09T06:16:14,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/c0c8bce224e34152a79cabd79bf27c04 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c0c8bce224e34152a79cabd79bf27c04 2024-12-09T06:16:14,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c0c8bce224e34152a79cabd79bf27c04, entries=150, sequenceid=64, filesize=11.7 K 2024-12-09T06:16:14,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c49fc2c2cb6f4d7d8b42133174c92b6b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c49fc2c2cb6f4d7d8b42133174c92b6b 2024-12-09T06:16:14,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c49fc2c2cb6f4d7d8b42133174c92b6b, entries=150, sequenceid=64, filesize=11.7 K 2024-12-09T06:16:14,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for edaa2b3dd1776fd6538daec178f1f68f in 462ms, sequenceid=64, compaction requested=true 2024-12-09T06:16:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:14,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-09T06:16:14,982 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:14,983 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:14,997 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:14,997 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/C is initiating minor compaction (all files) 2024-12-09T06:16:14,998 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/C in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,998 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c8623b538b58472491f0084e801d48be, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/ea907b96c6b546959c9f61203a7e1070, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c49fc2c2cb6f4d7d8b42133174c92b6b] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=35.2 K 2024-12-09T06:16:14,998 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:14,998 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/A is initiating minor compaction (all files) 2024-12-09T06:16:14,998 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/A in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,998 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/07889b634d7847d29a9a2ce26dd32c2d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/036ef63581fa4037862b828b487e5538, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/7486ef0d76774174a43aa2f3becc9ce2] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=115.9 K 2024-12-09T06:16:14,998 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:14,999 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/07889b634d7847d29a9a2ce26dd32c2d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/036ef63581fa4037862b828b487e5538, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/7486ef0d76774174a43aa2f3becc9ce2] 2024-12-09T06:16:15,002 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 07889b634d7847d29a9a2ce26dd32c2d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733724970894 2024-12-09T06:16:15,002 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8623b538b58472491f0084e801d48be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733724970894 2024-12-09T06:16:15,006 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea907b96c6b546959c9f61203a7e1070, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733724971083 2024-12-09T06:16:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:15,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T06:16:15,008 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 036ef63581fa4037862b828b487e5538, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733724971083 2024-12-09T06:16:15,012 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49fc2c2cb6f4d7d8b42133174c92b6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733724973348 2024-12-09T06:16:15,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:15,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:15,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:15,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:15,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:15,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:15,015 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7486ef0d76774174a43aa2f3becc9ce2, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733724973329 2024-12-09T06:16:15,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120979b559e64e684882913e5f9d64250572_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724974659/Put/seqid=0 2024-12-09T06:16:15,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725035041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725035044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,067 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:15,068 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/b976f3fdb5134583a80946c2eaa1cce4 is 50, key is test_row_0/C:col10/1733724974519/Put/seqid=0 2024-12-09T06:16:15,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:15,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725035029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725035048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,082 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T06:16:15,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741959_1135 (size=14594) 2024-12-09T06:16:15,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:15,142 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412096805419f2920407cb4297a10d7670689_edaa2b3dd1776fd6538daec178f1f68f store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:15,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741960_1136 (size=12104) 2024-12-09T06:16:15,158 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412096805419f2920407cb4297a10d7670689_edaa2b3dd1776fd6538daec178f1f68f, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:15,159 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096805419f2920407cb4297a10d7670689_edaa2b3dd1776fd6538daec178f1f68f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:15,161 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120979b559e64e684882913e5f9d64250572_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979b559e64e684882913e5f9d64250572_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:15,173 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/b72819f6156e4dbbb0ebacd0981430df, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:15,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/b72819f6156e4dbbb0ebacd0981430df is 175, key is test_row_0/A:col10/1733724974659/Put/seqid=0 2024-12-09T06:16:15,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725035170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,178 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/b976f3fdb5134583a80946c2eaa1cce4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/b976f3fdb5134583a80946c2eaa1cce4 2024-12-09T06:16:15,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725035178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725035177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725035188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,202 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/C of edaa2b3dd1776fd6538daec178f1f68f into b976f3fdb5134583a80946c2eaa1cce4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:15,202 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:15,202 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/C, priority=13, startTime=1733724974982; duration=0sec 2024-12-09T06:16:15,203 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:15,203 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:C 2024-12-09T06:16:15,203 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:15,205 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:15,205 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/B is initiating minor compaction (all files) 2024-12-09T06:16:15,206 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/B in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,206 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/4b6cffbc256547418b8be2f25589cfd9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/cf497e431e574e7987ac7349da8845c9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c0c8bce224e34152a79cabd79bf27c04] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=35.2 K 2024-12-09T06:16:15,207 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b6cffbc256547418b8be2f25589cfd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733724970894 2024-12-09T06:16:15,207 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf497e431e574e7987ac7349da8845c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733724971083 2024-12-09T06:16:15,207 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c8bce224e34152a79cabd79bf27c04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733724973348 2024-12-09T06:16:15,218 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#B#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:15,219 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/80976f146ba941c99c1e8a37d8b57515 is 50, key is test_row_0/B:col10/1733724974519/Put/seqid=0 2024-12-09T06:16:15,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725035226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:15,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:15,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741962_1138 (size=39549) 2024-12-09T06:16:15,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741961_1137 (size=4469) 2024-12-09T06:16:15,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741963_1139 (size=12104) 2024-12-09T06:16:15,283 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/80976f146ba941c99c1e8a37d8b57515 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/80976f146ba941c99c1e8a37d8b57515 2024-12-09T06:16:15,291 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/B of edaa2b3dd1776fd6538daec178f1f68f into 80976f146ba941c99c1e8a37d8b57515(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:15,291 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:15,291 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/B, priority=13, startTime=1733724974982; duration=0sec 2024-12-09T06:16:15,291 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:15,291 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:B 2024-12-09T06:16:15,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725035401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725035401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725035409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725035409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/b72819f6156e4dbbb0ebacd0981430df 2024-12-09T06:16:15,661 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#A#compaction#118 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:15,661 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/4b2371938bfa4dfc95218ca909ea7abc is 175, key is test_row_0/A:col10/1733724974519/Put/seqid=0 2024-12-09T06:16:15,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741964_1140 (size=31058) 2024-12-09T06:16:15,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/9513429e0fe34669b147edc632eb7da1 is 50, key is test_row_0/B:col10/1733724974659/Put/seqid=0 2024-12-09T06:16:15,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725035718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:15,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725035721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725035727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725035729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:15,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741965_1141 (size=12001) 2024-12-09T06:16:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725035737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:15,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:15,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:15,757 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/4b2371938bfa4dfc95218ca909ea7abc as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4b2371938bfa4dfc95218ca909ea7abc 2024-12-09T06:16:15,783 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/A of edaa2b3dd1776fd6538daec178f1f68f into 4b2371938bfa4dfc95218ca909ea7abc(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:15,783 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:15,783 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/A, priority=13, startTime=1733724974982; duration=0sec 2024-12-09T06:16:15,784 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:15,784 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:A 2024-12-09T06:16:15,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:15,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/9513429e0fe34669b147edc632eb7da1 2024-12-09T06:16:16,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/160626f900fc49d084d05498c2980fb1 is 50, key is test_row_0/C:col10/1733724974659/Put/seqid=0 2024-12-09T06:16:16,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741966_1142 (size=12001) 2024-12-09T06:16:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725036229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:16,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725036241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725036249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:16,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725036249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:16,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:16,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:16,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/160626f900fc49d084d05498c2980fb1 2024-12-09T06:16:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/b72819f6156e4dbbb0ebacd0981430df as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/b72819f6156e4dbbb0ebacd0981430df 2024-12-09T06:16:16,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:16,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/b72819f6156e4dbbb0ebacd0981430df, entries=200, sequenceid=101, filesize=38.6 K 2024-12-09T06:16:16,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/9513429e0fe34669b147edc632eb7da1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/9513429e0fe34669b147edc632eb7da1 2024-12-09T06:16:16,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725036758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:16,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/9513429e0fe34669b147edc632eb7da1, entries=150, sequenceid=101, filesize=11.7 K 2024-12-09T06:16:16,771 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:16,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/160626f900fc49d084d05498c2980fb1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/160626f900fc49d084d05498c2980fb1 2024-12-09T06:16:16,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/160626f900fc49d084d05498c2980fb1, entries=150, sequenceid=101, filesize=11.7 K 2024-12-09T06:16:16,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for edaa2b3dd1776fd6538daec178f1f68f in 1821ms, sequenceid=101, compaction requested=false 2024-12-09T06:16:16,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:16,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T06:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:16,859 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:16,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b3100450cab2476880a69f695eb3bf7e_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724975021/Put/seqid=0 2024-12-09T06:16:16,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741967_1143 (size=12154) 2024-12-09T06:16:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:16,951 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b3100450cab2476880a69f695eb3bf7e_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3100450cab2476880a69f695eb3bf7e_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/89a274368ca44de4a569dba2f5d4f24c, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/89a274368ca44de4a569dba2f5d4f24c is 175, key is test_row_0/A:col10/1733724975021/Put/seqid=0 2024-12-09T06:16:17,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741968_1144 (size=30955) 2024-12-09T06:16:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:17,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:17,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724977289, max timestamp=1733724977325Num uniques 859; 2024-12-09T06:16:17,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:17,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724977289, max timestamp=1733724977325Num uniques 859; 2024-12-09T06:16:17,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:17,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724977289, max timestamp=1733724977325Num uniques 859; 2024-12-09T06:16:17,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:17,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725037368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725037378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725037380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725037382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,402 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/89a274368ca44de4a569dba2f5d4f24c 2024-12-09T06:16:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/54d075d8104b4ee4a0dd0140acd912c9 is 50, key is test_row_0/B:col10/1733724975021/Put/seqid=0 2024-12-09T06:16:17,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741969_1145 (size=12001) 2024-12-09T06:16:17,469 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/54d075d8104b4ee4a0dd0140acd912c9 2024-12-09T06:16:17,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/f8b34b3284644c29b12b60d0940e8490 is 50, key is test_row_0/C:col10/1733724975021/Put/seqid=0 2024-12-09T06:16:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725037487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725037497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725037497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725037497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741970_1146 (size=12001) 2024-12-09T06:16:17,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725037697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725037707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725037720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725037721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:17,937 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/f8b34b3284644c29b12b60d0940e8490 2024-12-09T06:16:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/89a274368ca44de4a569dba2f5d4f24c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/89a274368ca44de4a569dba2f5d4f24c 2024-12-09T06:16:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725038007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,013 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/89a274368ca44de4a569dba2f5d4f24c, entries=150, sequenceid=114, filesize=30.2 K 2024-12-09T06:16:18,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/54d075d8104b4ee4a0dd0140acd912c9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/54d075d8104b4ee4a0dd0140acd912c9 2024-12-09T06:16:18,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725038019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,031 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/54d075d8104b4ee4a0dd0140acd912c9, entries=150, sequenceid=114, filesize=11.7 K 2024-12-09T06:16:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/f8b34b3284644c29b12b60d0940e8490 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/f8b34b3284644c29b12b60d0940e8490 2024-12-09T06:16:18,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725038037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725038041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,057 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/f8b34b3284644c29b12b60d0940e8490, entries=150, sequenceid=114, filesize=11.7 K 2024-12-09T06:16:18,061 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139285 for edaa2b3dd1776fd6538daec178f1f68f in 1202ms, sequenceid=114, compaction requested=true 2024-12-09T06:16:18,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:18,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:18,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-09T06:16:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-09T06:16:18,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-09T06:16:18,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 7.1720 sec 2024-12-09T06:16:18,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 7.1920 sec 2024-12-09T06:16:18,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-12-09T06:16:18,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:18,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:18,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:18,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:18,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:18,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:18,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725038562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725038579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725038579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096ec9e1e7cb6d482fa2a8b53535bb69aa_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724977379/Put/seqid=0 2024-12-09T06:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725038583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741971_1147 (size=12304) 2024-12-09T06:16:18,638 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:18,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096ec9e1e7cb6d482fa2a8b53535bb69aa_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096ec9e1e7cb6d482fa2a8b53535bb69aa_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:18,685 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/910747e5a78d405aa6b6dde1aa6ad850, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:18,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/910747e5a78d405aa6b6dde1aa6ad850 is 175, key is test_row_0/A:col10/1733724977379/Put/seqid=0 2024-12-09T06:16:18,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741972_1148 (size=31105) 2024-12-09T06:16:18,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/910747e5a78d405aa6b6dde1aa6ad850 2024-12-09T06:16:18,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725038697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725038697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725038697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/7f17a7a3e5594b9aa62231ee6079025a is 50, key is test_row_0/B:col10/1733724977379/Put/seqid=0 2024-12-09T06:16:18,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725038789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,797 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741973_1149 (size=12151) 2024-12-09T06:16:18,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725038907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725038909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725038909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,129 INFO [master/82ac79f0323b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T06:16:19,129 INFO [master/82ac79f0323b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T06:16:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T06:16:19,137 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:19,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-09T06:16:19,140 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:19,141 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:19,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T06:16:19,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/7f17a7a3e5594b9aa62231ee6079025a 2024-12-09T06:16:19,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725039216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725039216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725039227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/99d55289380b489e8c7e253ee0285fb8 is 50, key is test_row_0/C:col10/1733724977379/Put/seqid=0 2024-12-09T06:16:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T06:16:19,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741974_1150 (size=12151) 2024-12-09T06:16:19,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:19,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:19,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:19,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:19,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T06:16:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725039596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,601 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/99d55289380b489e8c7e253ee0285fb8 2024-12-09T06:16:19,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/910747e5a78d405aa6b6dde1aa6ad850 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/910747e5a78d405aa6b6dde1aa6ad850 2024-12-09T06:16:19,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/910747e5a78d405aa6b6dde1aa6ad850, entries=150, sequenceid=151, filesize=30.4 K 2024-12-09T06:16:19,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/7f17a7a3e5594b9aa62231ee6079025a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/7f17a7a3e5594b9aa62231ee6079025a 2024-12-09T06:16:19,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/7f17a7a3e5594b9aa62231ee6079025a, entries=150, sequenceid=151, filesize=11.9 K 2024-12-09T06:16:19,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/99d55289380b489e8c7e253ee0285fb8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/99d55289380b489e8c7e253ee0285fb8 2024-12-09T06:16:19,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/99d55289380b489e8c7e253ee0285fb8, entries=150, sequenceid=151, filesize=11.9 K 2024-12-09T06:16:19,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~142.73 KB/146155, heapSize ~404.34 KB/414040, currentSize=46.96 KB/48090 for edaa2b3dd1776fd6538daec178f1f68f in 1163ms, sequenceid=151, compaction requested=true 2024-12-09T06:16:19,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:19,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:19,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:19,698 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:19,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:19,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:19,698 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:19,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:19,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:19,700 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:19,700 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132667 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:19,701 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/A is initiating minor compaction (all files) 2024-12-09T06:16:19,701 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/A in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,701 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4b2371938bfa4dfc95218ca909ea7abc, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/b72819f6156e4dbbb0ebacd0981430df, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/89a274368ca44de4a569dba2f5d4f24c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/910747e5a78d405aa6b6dde1aa6ad850] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=129.6 K 2024-12-09T06:16:19,701 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,701 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4b2371938bfa4dfc95218ca909ea7abc, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/b72819f6156e4dbbb0ebacd0981430df, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/89a274368ca44de4a569dba2f5d4f24c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/910747e5a78d405aa6b6dde1aa6ad850] 2024-12-09T06:16:19,701 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/B is initiating minor compaction (all files) 2024-12-09T06:16:19,701 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/B in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,702 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/80976f146ba941c99c1e8a37d8b57515, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/9513429e0fe34669b147edc632eb7da1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/54d075d8104b4ee4a0dd0140acd912c9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/7f17a7a3e5594b9aa62231ee6079025a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=47.1 K 2024-12-09T06:16:19,703 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2371938bfa4dfc95218ca909ea7abc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733724973348 2024-12-09T06:16:19,703 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 80976f146ba941c99c1e8a37d8b57515, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733724973348 2024-12-09T06:16:19,705 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72819f6156e4dbbb0ebacd0981430df, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733724974603 2024-12-09T06:16:19,705 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9513429e0fe34669b147edc632eb7da1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733724974659 2024-12-09T06:16:19,713 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89a274368ca44de4a569dba2f5d4f24c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733724975021 2024-12-09T06:16:19,713 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d075d8104b4ee4a0dd0140acd912c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733724975021 2024-12-09T06:16:19,714 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 910747e5a78d405aa6b6dde1aa6ad850, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733724977375 2024-12-09T06:16:19,714 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f17a7a3e5594b9aa62231ee6079025a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733724977375 2024-12-09T06:16:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:19,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T06:16:19,743 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#B#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:19,744 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/60cd88b1f44c44e099b40c129157909b is 50, key is test_row_0/B:col10/1733724977379/Put/seqid=0 2024-12-09T06:16:19,745 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:19,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:19,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:19,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:19,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:19,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:19,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:19,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T06:16:19,787 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209efcccd027c91477a9e9ca494d1731308_edaa2b3dd1776fd6538daec178f1f68f store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:19,791 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209efcccd027c91477a9e9ca494d1731308_edaa2b3dd1776fd6538daec178f1f68f, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:19,791 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209efcccd027c91477a9e9ca494d1731308_edaa2b3dd1776fd6538daec178f1f68f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:19,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741975_1151 (size=12391) 2024-12-09T06:16:19,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120935afada78e1042a185f146a1faaf702a_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724979740/Put/seqid=0 2024-12-09T06:16:19,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741976_1152 (size=4469) 2024-12-09T06:16:19,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741977_1153 (size=12304) 2024-12-09T06:16:19,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:19,838 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120935afada78e1042a185f146a1faaf702a_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935afada78e1042a185f146a1faaf702a_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:19,839 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/cb5a4badc16c40058027b1b815a3e2a1, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:19,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/cb5a4badc16c40058027b1b815a3e2a1 is 175, key is test_row_0/A:col10/1733724979740/Put/seqid=0 2024-12-09T06:16:19,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725039828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725039844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725039845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741978_1154 (size=31105) 2024-12-09T06:16:19,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:19,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:19,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725039947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725039957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725039957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:20,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:20,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725040157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:20,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725040177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:20,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725040178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:20,223 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/60cd88b1f44c44e099b40c129157909b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/60cd88b1f44c44e099b40c129157909b 2024-12-09T06:16:20,224 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#A#compaction#129 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:20,225 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/eb0dcb53050e4283af0d2b2ae7cfd5f3 is 175, key is test_row_0/A:col10/1733724977379/Put/seqid=0 2024-12-09T06:16:20,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:20,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,231 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/B of edaa2b3dd1776fd6538daec178f1f68f into 60cd88b1f44c44e099b40c129157909b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:20,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:20,231 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:20,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,231 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/B, priority=12, startTime=1733724979698; duration=0sec 2024-12-09T06:16:20,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,232 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:20,232 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:B 2024-12-09T06:16:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,232 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,239 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:20,239 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/C is initiating minor compaction (all files) 2024-12-09T06:16:20,239 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/C in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,239 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/b976f3fdb5134583a80946c2eaa1cce4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/160626f900fc49d084d05498c2980fb1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/f8b34b3284644c29b12b60d0940e8490, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/99d55289380b489e8c7e253ee0285fb8] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=47.1 K 2024-12-09T06:16:20,239 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b976f3fdb5134583a80946c2eaa1cce4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733724973348 2024-12-09T06:16:20,240 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 160626f900fc49d084d05498c2980fb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733724974659 2024-12-09T06:16:20,240 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b34b3284644c29b12b60d0940e8490, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733724975021 2024-12-09T06:16:20,240 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 99d55289380b489e8c7e253ee0285fb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733724977375 2024-12-09T06:16:20,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741979_1155 (size=31345) 2024-12-09T06:16:20,255 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:20,256 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c79d6e7f55ee438ab0b3869a7c4e64f4 is 50, key is test_row_0/C:col10/1733724977379/Put/seqid=0 2024-12-09T06:16:20,258 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/cb5a4badc16c40058027b1b815a3e2a1 2024-12-09T06:16:20,261 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/eb0dcb53050e4283af0d2b2ae7cfd5f3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/eb0dcb53050e4283af0d2b2ae7cfd5f3 2024-12-09T06:16:20,271 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/A of edaa2b3dd1776fd6538daec178f1f68f into eb0dcb53050e4283af0d2b2ae7cfd5f3(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:20,271 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:20,272 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/A, priority=12, startTime=1733724979698; duration=0sec 2024-12-09T06:16:20,272 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:20,272 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:A 2024-12-09T06:16:20,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/c836c66da4a24763a8f9c44b0e176b81 is 50, key is test_row_0/B:col10/1733724979740/Put/seqid=0 2024-12-09T06:16:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T06:16:20,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741980_1156 (size=12391) 2024-12-09T06:16:20,315 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c79d6e7f55ee438ab0b3869a7c4e64f4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c79d6e7f55ee438ab0b3869a7c4e64f4 2024-12-09T06:16:20,323 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/C of edaa2b3dd1776fd6538daec178f1f68f into c79d6e7f55ee438ab0b3869a7c4e64f4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:20,323 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:20,323 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/C, priority=12, startTime=1733724979698; duration=0sec 2024-12-09T06:16:20,324 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:20,324 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:C 2024-12-09T06:16:20,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741981_1157 (size=12151) 2024-12-09T06:16:20,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/c836c66da4a24763a8f9c44b0e176b81 2024-12-09T06:16:20,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/3b2a9b8a7fcc422bb3faba1140df228b is 50, key is test_row_0/C:col10/1733724979740/Put/seqid=0 2024-12-09T06:16:20,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:20,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:20,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741982_1158 (size=12151) 2024-12-09T06:16:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725040467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:20,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725040497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:20,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:20,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725040497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:20,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:20,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:20,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:20,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/3b2a9b8a7fcc422bb3faba1140df228b 2024-12-09T06:16:20,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/cb5a4badc16c40058027b1b815a3e2a1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cb5a4badc16c40058027b1b815a3e2a1 2024-12-09T06:16:20,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cb5a4badc16c40058027b1b815a3e2a1, entries=150, sequenceid=164, filesize=30.4 K 2024-12-09T06:16:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/c836c66da4a24763a8f9c44b0e176b81 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c836c66da4a24763a8f9c44b0e176b81 2024-12-09T06:16:20,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c836c66da4a24763a8f9c44b0e176b81, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T06:16:20,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/3b2a9b8a7fcc422bb3faba1140df228b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3b2a9b8a7fcc422bb3faba1140df228b 2024-12-09T06:16:20,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:20,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:20,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3b2a9b8a7fcc422bb3faba1140df228b, entries=150, sequenceid=164, filesize=11.9 K 2024-12-09T06:16:20,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for edaa2b3dd1776fd6538daec178f1f68f in 1118ms, sequenceid=164, compaction requested=false 2024-12-09T06:16:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:20,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T06:16:20,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:21,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ac227835b4b54205a5bd9e1bb7372dbc_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724980978/Put/seqid=0 2024-12-09T06:16:21,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725041017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725041019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725041019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741983_1159 (size=12304) 2024-12-09T06:16:21,043 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:21,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ac227835b4b54205a5bd9e1bb7372dbc_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ac227835b4b54205a5bd9e1bb7372dbc_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:21,069 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/d9ee816dd71f4bb69870ee55e3b80b6a, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:21,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/d9ee816dd71f4bb69870ee55e3b80b6a is 175, key is test_row_0/A:col10/1733724980978/Put/seqid=0 2024-12-09T06:16:21,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741984_1160 (size=31105) 2024-12-09T06:16:21,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/d9ee816dd71f4bb69870ee55e3b80b6a 2024-12-09T06:16:21,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/99271baa31e643d582f8993d469167d8 is 50, key is test_row_0/B:col10/1733724980978/Put/seqid=0 2024-12-09T06:16:21,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741985_1161 (size=12151) 2024-12-09T06:16:21,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725041127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725041129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T06:16:21,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725041337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725041341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/99271baa31e643d582f8993d469167d8 2024-12-09T06:16:21,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/4421a2c5235b4d248db6891940e67d64 is 50, key is test_row_0/C:col10/1733724980978/Put/seqid=0 2024-12-09T06:16:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741986_1162 (size=12151) 2024-12-09T06:16:21,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/4421a2c5235b4d248db6891940e67d64 2024-12-09T06:16:21,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/d9ee816dd71f4bb69870ee55e3b80b6a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d9ee816dd71f4bb69870ee55e3b80b6a 2024-12-09T06:16:21,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:21,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:21,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:21,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725041627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,638 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725041647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d9ee816dd71f4bb69870ee55e3b80b6a, entries=150, sequenceid=188, filesize=30.4 K 2024-12-09T06:16:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/99271baa31e643d582f8993d469167d8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/99271baa31e643d582f8993d469167d8 2024-12-09T06:16:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725041661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:21,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/99271baa31e643d582f8993d469167d8, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T06:16:21,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/4421a2c5235b4d248db6891940e67d64 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4421a2c5235b4d248db6891940e67d64 2024-12-09T06:16:21,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4421a2c5235b4d248db6891940e67d64, entries=150, sequenceid=188, filesize=11.9 K 2024-12-09T06:16:21,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for edaa2b3dd1776fd6538daec178f1f68f in 706ms, sequenceid=188, compaction requested=true 2024-12-09T06:16:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:21,688 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:21,688 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:21,689 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:21,689 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:21,689 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/B is initiating minor compaction (all files) 2024-12-09T06:16:21,689 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/A is initiating minor compaction (all files) 2024-12-09T06:16:21,689 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/B in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,689 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/A in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,689 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/eb0dcb53050e4283af0d2b2ae7cfd5f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cb5a4badc16c40058027b1b815a3e2a1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d9ee816dd71f4bb69870ee55e3b80b6a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=91.4 K 2024-12-09T06:16:21,689 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/60cd88b1f44c44e099b40c129157909b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c836c66da4a24763a8f9c44b0e176b81, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/99271baa31e643d582f8993d469167d8] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=35.8 K 2024-12-09T06:16:21,690 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,690 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/eb0dcb53050e4283af0d2b2ae7cfd5f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cb5a4badc16c40058027b1b815a3e2a1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d9ee816dd71f4bb69870ee55e3b80b6a] 2024-12-09T06:16:21,690 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 60cd88b1f44c44e099b40c129157909b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733724977375 2024-12-09T06:16:21,690 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb0dcb53050e4283af0d2b2ae7cfd5f3, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733724977375 2024-12-09T06:16:21,691 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c836c66da4a24763a8f9c44b0e176b81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733724978571 2024-12-09T06:16:21,691 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb5a4badc16c40058027b1b815a3e2a1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733724978571 2024-12-09T06:16:21,691 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9ee816dd71f4bb69870ee55e3b80b6a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733724979823 2024-12-09T06:16:21,691 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 99271baa31e643d582f8993d469167d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733724979823 2024-12-09T06:16:21,709 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:21,723 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:21,723 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/8b60cebe6ca7422e80bf0e2a1a6bbe3f is 50, key is test_row_0/B:col10/1733724980978/Put/seqid=0 2024-12-09T06:16:21,745 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120966f433cdb5b84eeaafd3827126a82782_edaa2b3dd1776fd6538daec178f1f68f store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:21,748 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120966f433cdb5b84eeaafd3827126a82782_edaa2b3dd1776fd6538daec178f1f68f, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:21,748 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120966f433cdb5b84eeaafd3827126a82782_edaa2b3dd1776fd6538daec178f1f68f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:21,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T06:16:21,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:21,781 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:16:21,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:21,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:21,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:21,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:21,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:21,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:21,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741987_1163 (size=12493) 2024-12-09T06:16:21,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741988_1164 (size=4469) 2024-12-09T06:16:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f27e4cae3c5e49aa83186b7b6a739510_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724981017/Put/seqid=0 2024-12-09T06:16:21,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741989_1165 (size=12304) 2024-12-09T06:16:22,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:22,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724982107, max timestamp=1733724982159Num uniques 859; 2024-12-09T06:16:22,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724982107, max timestamp=1733724982159Num uniques 859; 2024-12-09T06:16:22,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724982107, max timestamp=1733724982159Num uniques 859; 2024-12-09T06:16:22,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:22,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:22,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725042211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725042212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725042212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,220 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/8b60cebe6ca7422e80bf0e2a1a6bbe3f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/8b60cebe6ca7422e80bf0e2a1a6bbe3f 2024-12-09T06:16:22,232 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#A#compaction#137 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:22,233 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/86ca7166bd0345dcb38953967d8a2f76 is 175, key is test_row_0/A:col10/1733724980978/Put/seqid=0 2024-12-09T06:16:22,236 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/B of edaa2b3dd1776fd6538daec178f1f68f into 8b60cebe6ca7422e80bf0e2a1a6bbe3f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:22,236 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:22,237 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/B, priority=13, startTime=1733724981688; duration=0sec 2024-12-09T06:16:22,237 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:22,237 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:B 2024-12-09T06:16:22,237 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:22,241 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:22,242 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/C is initiating minor compaction (all files) 2024-12-09T06:16:22,242 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/C in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:22,242 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c79d6e7f55ee438ab0b3869a7c4e64f4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3b2a9b8a7fcc422bb3faba1140df228b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4421a2c5235b4d248db6891940e67d64] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=35.8 K 2024-12-09T06:16:22,242 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c79d6e7f55ee438ab0b3869a7c4e64f4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733724977375 2024-12-09T06:16:22,243 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b2a9b8a7fcc422bb3faba1140df228b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733724978571 2024-12-09T06:16:22,243 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4421a2c5235b4d248db6891940e67d64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733724979823 2024-12-09T06:16:22,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741990_1166 (size=31447) 2024-12-09T06:16:22,260 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:22,261 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/3887fa2708544574b5e489bcbc9adec3 is 50, key is test_row_0/C:col10/1733724980978/Put/seqid=0 2024-12-09T06:16:22,266 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/86ca7166bd0345dcb38953967d8a2f76 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/86ca7166bd0345dcb38953967d8a2f76 2024-12-09T06:16:22,273 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/A of edaa2b3dd1776fd6538daec178f1f68f into 86ca7166bd0345dcb38953967d8a2f76(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:22,274 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:22,274 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/A, priority=13, startTime=1733724981688; duration=0sec 2024-12-09T06:16:22,274 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:22,274 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:A 2024-12-09T06:16:22,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:22,286 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f27e4cae3c5e49aa83186b7b6a739510_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f27e4cae3c5e49aa83186b7b6a739510_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:22,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/293e93b1bc1b4dbc8c1ea520e6c40da9, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:22,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/293e93b1bc1b4dbc8c1ea520e6c40da9 is 175, key is test_row_0/A:col10/1733724981017/Put/seqid=0 2024-12-09T06:16:22,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741991_1167 (size=12493) 2024-12-09T06:16:22,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741992_1168 (size=31105) 2024-12-09T06:16:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725042316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725042327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725042327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725042527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725042536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725042537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,697 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/3887fa2708544574b5e489bcbc9adec3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3887fa2708544574b5e489bcbc9adec3 2024-12-09T06:16:22,703 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/C of edaa2b3dd1776fd6538daec178f1f68f into 3887fa2708544574b5e489bcbc9adec3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:22,703 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:22,703 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/C, priority=13, startTime=1733724981688; duration=0sec 2024-12-09T06:16:22,703 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:22,703 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:C 2024-12-09T06:16:22,706 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/293e93b1bc1b4dbc8c1ea520e6c40da9 2024-12-09T06:16:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/3e0ba2bfc6754bd4b52172b23765298c is 50, key is test_row_0/B:col10/1733724981017/Put/seqid=0 2024-12-09T06:16:22,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741993_1169 (size=12151) 2024-12-09T06:16:22,743 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/3e0ba2bfc6754bd4b52172b23765298c 2024-12-09T06:16:22,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/1fb3967191e741f8ba6450818e8f60e7 is 50, key is test_row_0/C:col10/1733724981017/Put/seqid=0 2024-12-09T06:16:22,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741994_1170 (size=12151) 2024-12-09T06:16:22,812 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/1fb3967191e741f8ba6450818e8f60e7 2024-12-09T06:16:22,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725042837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725042846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725042847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/293e93b1bc1b4dbc8c1ea520e6c40da9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/293e93b1bc1b4dbc8c1ea520e6c40da9 2024-12-09T06:16:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49238 deadline: 1733725042847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,851 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:22,882 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/293e93b1bc1b4dbc8c1ea520e6c40da9, entries=150, sequenceid=199, filesize=30.4 K 2024-12-09T06:16:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/3e0ba2bfc6754bd4b52172b23765298c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/3e0ba2bfc6754bd4b52172b23765298c 2024-12-09T06:16:22,902 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/3e0ba2bfc6754bd4b52172b23765298c, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T06:16:22,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/1fb3967191e741f8ba6450818e8f60e7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1fb3967191e741f8ba6450818e8f60e7 2024-12-09T06:16:22,927 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1fb3967191e741f8ba6450818e8f60e7, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T06:16:22,932 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for edaa2b3dd1776fd6538daec178f1f68f in 1151ms, sequenceid=199, compaction requested=false 2024-12-09T06:16:22,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:22,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:22,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-09T06:16:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-09T06:16:22,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-09T06:16:22,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7920 sec 2024-12-09T06:16:22,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 3.8010 sec 2024-12-09T06:16:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T06:16:23,307 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:23,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-09T06:16:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T06:16:23,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:23,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:23,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:23,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-09T06:16:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:23,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c6213d50558b40abacc49e45d27d957b_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724983347/Put/seqid=0 2024-12-09T06:16:23,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725043362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725043365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725043365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741995_1171 (size=14794) 2024-12-09T06:16:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T06:16:23,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T06:16:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:23,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725043477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725043477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725043477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T06:16:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:23,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T06:16:23,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725043686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725043687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725043697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:23,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:23,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T06:16:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:23,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,781 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c6213d50558b40abacc49e45d27d957b_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c6213d50558b40abacc49e45d27d957b_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:23,787 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/cdc26ba3344d41a0837c5ef26a9ec6c9, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:23,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/cdc26ba3344d41a0837c5ef26a9ec6c9 is 175, key is test_row_0/A:col10/1733724983347/Put/seqid=0 2024-12-09T06:16:23,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741996_1172 (size=39749) 2024-12-09T06:16:23,796 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/cdc26ba3344d41a0837c5ef26a9ec6c9 2024-12-09T06:16:23,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/860fc8e902184b2fae433f489b697144 is 50, key is test_row_0/B:col10/1733724983347/Put/seqid=0 2024-12-09T06:16:23,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741997_1173 (size=12151) 2024-12-09T06:16:23,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/860fc8e902184b2fae433f489b697144 2024-12-09T06:16:23,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/de05586dcccf4cde9fdd806626c371d4 is 50, key is test_row_0/C:col10/1733724983347/Put/seqid=0 2024-12-09T06:16:23,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741998_1174 (size=12151) 2024-12-09T06:16:23,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T06:16:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T06:16:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:23,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725043997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725044007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725044007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T06:16:24,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:24,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T06:16:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:24,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:24,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:24,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/de05586dcccf4cde9fdd806626c371d4 2024-12-09T06:16:24,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/cdc26ba3344d41a0837c5ef26a9ec6c9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cdc26ba3344d41a0837c5ef26a9ec6c9 2024-12-09T06:16:24,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cdc26ba3344d41a0837c5ef26a9ec6c9, entries=200, sequenceid=238, filesize=38.8 K 2024-12-09T06:16:24,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/860fc8e902184b2fae433f489b697144 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/860fc8e902184b2fae433f489b697144 2024-12-09T06:16:24,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/860fc8e902184b2fae433f489b697144, entries=150, sequenceid=238, filesize=11.9 K 2024-12-09T06:16:24,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/de05586dcccf4cde9fdd806626c371d4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/de05586dcccf4cde9fdd806626c371d4 2024-12-09T06:16:24,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/de05586dcccf4cde9fdd806626c371d4, entries=150, sequenceid=238, filesize=11.9 K 2024-12-09T06:16:24,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for edaa2b3dd1776fd6538daec178f1f68f in 968ms, sequenceid=238, compaction requested=true 2024-12-09T06:16:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:24,316 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:24,316 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:24,322 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:24,322 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/A is initiating minor compaction (all files) 2024-12-09T06:16:24,322 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/A in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,322 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:24,322 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/B is initiating minor compaction (all files) 2024-12-09T06:16:24,322 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/86ca7166bd0345dcb38953967d8a2f76, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/293e93b1bc1b4dbc8c1ea520e6c40da9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cdc26ba3344d41a0837c5ef26a9ec6c9] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=99.9 K 2024-12-09T06:16:24,322 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/B in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,322 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,322 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/8b60cebe6ca7422e80bf0e2a1a6bbe3f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/3e0ba2bfc6754bd4b52172b23765298c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/860fc8e902184b2fae433f489b697144] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=35.9 K 2024-12-09T06:16:24,322 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/86ca7166bd0345dcb38953967d8a2f76, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/293e93b1bc1b4dbc8c1ea520e6c40da9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cdc26ba3344d41a0837c5ef26a9ec6c9] 2024-12-09T06:16:24,324 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b60cebe6ca7422e80bf0e2a1a6bbe3f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733724979823 2024-12-09T06:16:24,324 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86ca7166bd0345dcb38953967d8a2f76, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733724979823 2024-12-09T06:16:24,325 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e0ba2bfc6754bd4b52172b23765298c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733724981007 2024-12-09T06:16:24,325 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 293e93b1bc1b4dbc8c1ea520e6c40da9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733724981007 2024-12-09T06:16:24,325 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 860fc8e902184b2fae433f489b697144, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733724982186 2024-12-09T06:16:24,325 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdc26ba3344d41a0837c5ef26a9ec6c9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733724982186 2024-12-09T06:16:24,338 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:24,342 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209213079a989624e16975ecb15b1ed2350_edaa2b3dd1776fd6538daec178f1f68f store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:24,342 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:24,342 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/24dce6f9d4404ccf9d1a7289ffe2abbf is 50, key is test_row_0/B:col10/1733724983347/Put/seqid=0 2024-12-09T06:16:24,343 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209213079a989624e16975ecb15b1ed2350_edaa2b3dd1776fd6538daec178f1f68f, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:24,343 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209213079a989624e16975ecb15b1ed2350_edaa2b3dd1776fd6538daec178f1f68f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:24,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742000_1176 (size=12595) 2024-12-09T06:16:24,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T06:16:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,395 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:24,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741999_1175 (size=4469) 2024-12-09T06:16:24,402 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#A#compaction#146 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:24,402 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/a917c706a0cd4937835ae77d5591acdd is 175, key is test_row_0/A:col10/1733724983347/Put/seqid=0 2024-12-09T06:16:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091f0a1c8dd2c24e9daea30a549a18d860_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724983362/Put/seqid=0 2024-12-09T06:16:24,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742001_1177 (size=31549) 2024-12-09T06:16:24,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742002_1178 (size=12304) 2024-12-09T06:16:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T06:16:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:24,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:24,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724984531, max timestamp=1733724984548Num uniques 859; 2024-12-09T06:16:24,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733724984531, max timestamp=1733724984548Num uniques 859; 2024-12-09T06:16:24,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:24,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:24,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724984531, max timestamp=1733724984548Num uniques 859; 2024-12-09T06:16:24,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:24,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725044587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725044588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725044589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725044707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725044708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725044708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,800 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/24dce6f9d4404ccf9d1a7289ffe2abbf as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24dce6f9d4404ccf9d1a7289ffe2abbf 2024-12-09T06:16:24,816 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/B of edaa2b3dd1776fd6538daec178f1f68f into 24dce6f9d4404ccf9d1a7289ffe2abbf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:24,816 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:24,816 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/B, priority=13, startTime=1733724984316; duration=0sec 2024-12-09T06:16:24,816 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:24,817 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:B 2024-12-09T06:16:24,817 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:24,819 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:24,819 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/C is initiating minor compaction (all files) 2024-12-09T06:16:24,819 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/C in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:24,819 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3887fa2708544574b5e489bcbc9adec3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1fb3967191e741f8ba6450818e8f60e7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/de05586dcccf4cde9fdd806626c371d4] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=35.9 K 2024-12-09T06:16:24,820 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/a917c706a0cd4937835ae77d5591acdd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a917c706a0cd4937835ae77d5591acdd 2024-12-09T06:16:24,820 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3887fa2708544574b5e489bcbc9adec3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733724979823 2024-12-09T06:16:24,820 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb3967191e741f8ba6450818e8f60e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733724981007 2024-12-09T06:16:24,822 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting de05586dcccf4cde9fdd806626c371d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733724982186 2024-12-09T06:16:24,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:24,833 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091f0a1c8dd2c24e9daea30a549a18d860_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091f0a1c8dd2c24e9daea30a549a18d860_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:24,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/a54f17f8f2da4cf5b7afe7f43d534383, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:24,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/a54f17f8f2da4cf5b7afe7f43d534383 is 175, key is test_row_0/A:col10/1733724983362/Put/seqid=0 2024-12-09T06:16:24,836 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:24,836 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c982aefae4ca452aa4b7ae6c4bb1d61a is 50, key is test_row_0/C:col10/1733724983347/Put/seqid=0 2024-12-09T06:16:24,844 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/A of edaa2b3dd1776fd6538daec178f1f68f into a917c706a0cd4937835ae77d5591acdd(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:24,844 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:24,844 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/A, priority=13, startTime=1733724984316; duration=0sec 2024-12-09T06:16:24,844 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:24,844 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:A 2024-12-09T06:16:24,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742003_1179 (size=31105) 2024-12-09T06:16:24,868 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/a54f17f8f2da4cf5b7afe7f43d534383 2024-12-09T06:16:24,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742004_1180 (size=12595) 2024-12-09T06:16:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/619fdfd34b834de1a1318d8b6706c992 is 50, key is test_row_0/B:col10/1733724983362/Put/seqid=0 2024-12-09T06:16:24,907 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/c982aefae4ca452aa4b7ae6c4bb1d61a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c982aefae4ca452aa4b7ae6c4bb1d61a 2024-12-09T06:16:24,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725044917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725044917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725044921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:24,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742005_1181 (size=12151) 2024-12-09T06:16:24,942 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/C of edaa2b3dd1776fd6538daec178f1f68f into c982aefae4ca452aa4b7ae6c4bb1d61a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:24,942 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:24,942 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/C, priority=13, startTime=1733724984316; duration=0sec 2024-12-09T06:16:24,942 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:24,942 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:C 2024-12-09T06:16:24,945 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/619fdfd34b834de1a1318d8b6706c992 2024-12-09T06:16:24,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/030efb05c4cd4239aaa4f93e80a9c360 is 50, key is test_row_0/C:col10/1733724983362/Put/seqid=0 2024-12-09T06:16:25,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742006_1182 (size=12151) 2024-12-09T06:16:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725045226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:25,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725045229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:25,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725045237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:25,453 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/030efb05c4cd4239aaa4f93e80a9c360 2024-12-09T06:16:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T06:16:25,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/a54f17f8f2da4cf5b7afe7f43d534383 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a54f17f8f2da4cf5b7afe7f43d534383 2024-12-09T06:16:25,487 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a54f17f8f2da4cf5b7afe7f43d534383, entries=150, sequenceid=248, filesize=30.4 K 2024-12-09T06:16:25,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/619fdfd34b834de1a1318d8b6706c992 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/619fdfd34b834de1a1318d8b6706c992 2024-12-09T06:16:25,511 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/619fdfd34b834de1a1318d8b6706c992, entries=150, sequenceid=248, filesize=11.9 K 2024-12-09T06:16:25,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/030efb05c4cd4239aaa4f93e80a9c360 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/030efb05c4cd4239aaa4f93e80a9c360 2024-12-09T06:16:25,523 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/030efb05c4cd4239aaa4f93e80a9c360, entries=150, sequenceid=248, filesize=11.9 K 2024-12-09T06:16:25,524 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.02 KB/139285 for edaa2b3dd1776fd6538daec178f1f68f in 1130ms, sequenceid=248, compaction requested=false 2024-12-09T06:16:25,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:25,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:25,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-09T06:16:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-09T06:16:25,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-09T06:16:25,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-12-09T06:16:25,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2270 sec 2024-12-09T06:16:25,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-12-09T06:16:25,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:25,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:25,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:25,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:25,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:25,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:25,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120929a70e50e2b84f5d9af6b05162fc907f_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724985684/Put/seqid=0 2024-12-09T06:16:25,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742007_1183 (size=14994) 2024-12-09T06:16:25,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725045740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:25,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725045745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:25,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725045747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:25,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725045757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:25,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725045857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725045857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725046067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725046067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:26,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:26,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120929a70e50e2b84f5d9af6b05162fc907f_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120929a70e50e2b84f5d9af6b05162fc907f_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:26,146 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/e4653695a93a4a38a08a0ed223773080, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:26,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/e4653695a93a4a38a08a0ed223773080 is 175, key is test_row_0/A:col10/1733724985684/Put/seqid=0 2024-12-09T06:16:26,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742008_1184 (size=39949) 2024-12-09T06:16:26,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:26,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725046379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725046379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:26,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:26,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/e4653695a93a4a38a08a0ed223773080 2024-12-09T06:16:26,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/aa9eb03ea3cb481985831fa7f68c9580 is 50, key is test_row_0/B:col10/1733724985684/Put/seqid=0 2024-12-09T06:16:26,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742009_1185 (size=12301) 2024-12-09T06:16:26,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725046769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:26,774 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:26,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725046776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:26,780 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:26,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725046899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725046899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:27,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/aa9eb03ea3cb481985831fa7f68c9580 2024-12-09T06:16:27,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/3586ac947f3d43d69ba2317a86c53414 is 50, key is test_row_0/C:col10/1733724985684/Put/seqid=0 2024-12-09T06:16:27,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742010_1186 (size=12301) 2024-12-09T06:16:27,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/3586ac947f3d43d69ba2317a86c53414 2024-12-09T06:16:27,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/e4653695a93a4a38a08a0ed223773080 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/e4653695a93a4a38a08a0ed223773080 2024-12-09T06:16:27,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/e4653695a93a4a38a08a0ed223773080, entries=200, sequenceid=288, filesize=39.0 K 2024-12-09T06:16:27,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/aa9eb03ea3cb481985831fa7f68c9580 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/aa9eb03ea3cb481985831fa7f68c9580 2024-12-09T06:16:27,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/aa9eb03ea3cb481985831fa7f68c9580, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T06:16:27,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/3586ac947f3d43d69ba2317a86c53414 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3586ac947f3d43d69ba2317a86c53414 2024-12-09T06:16:27,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3586ac947f3d43d69ba2317a86c53414, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T06:16:27,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~142.73 KB/146155, heapSize ~404.34 KB/414040, currentSize=40.25 KB/41220 for edaa2b3dd1776fd6538daec178f1f68f in 1553ms, sequenceid=288, compaction requested=true 2024-12-09T06:16:27,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:27,239 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:27,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:27,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:27,241 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:27,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:27,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:27,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store edaa2b3dd1776fd6538daec178f1f68f:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:27,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:27,249 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:27,249 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/B is initiating minor compaction (all files) 2024-12-09T06:16:27,249 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:27,249 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/B in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:27,249 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/A is initiating minor compaction (all files) 2024-12-09T06:16:27,249 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/A in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:27,249 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24dce6f9d4404ccf9d1a7289ffe2abbf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/619fdfd34b834de1a1318d8b6706c992, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/aa9eb03ea3cb481985831fa7f68c9580] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=36.2 K 2024-12-09T06:16:27,250 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a917c706a0cd4937835ae77d5591acdd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a54f17f8f2da4cf5b7afe7f43d534383, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/e4653695a93a4a38a08a0ed223773080] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=100.2 K 2024-12-09T06:16:27,250 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:27,250 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a917c706a0cd4937835ae77d5591acdd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a54f17f8f2da4cf5b7afe7f43d534383, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/e4653695a93a4a38a08a0ed223773080] 2024-12-09T06:16:27,250 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 24dce6f9d4404ccf9d1a7289ffe2abbf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733724982186 2024-12-09T06:16:27,250 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a917c706a0cd4937835ae77d5591acdd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733724982186 2024-12-09T06:16:27,251 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54f17f8f2da4cf5b7afe7f43d534383, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733724983361 2024-12-09T06:16:27,251 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 619fdfd34b834de1a1318d8b6706c992, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733724983361 2024-12-09T06:16:27,251 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4653695a93a4a38a08a0ed223773080, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733724984579 2024-12-09T06:16:27,251 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9eb03ea3cb481985831fa7f68c9580, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733724984586 2024-12-09T06:16:27,270 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#B#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:27,271 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/862aaae3646e4a1fb60fbea444557849 is 50, key is test_row_0/B:col10/1733724985684/Put/seqid=0 2024-12-09T06:16:27,273 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:27,305 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120930a3e98ca41e41a5bee6b7fd79459b49_edaa2b3dd1776fd6538daec178f1f68f store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:27,307 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120930a3e98ca41e41a5bee6b7fd79459b49_edaa2b3dd1776fd6538daec178f1f68f, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:27,307 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120930a3e98ca41e41a5bee6b7fd79459b49_edaa2b3dd1776fd6538daec178f1f68f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742011_1187 (size=12847) 2024-12-09T06:16:27,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742012_1188 (size=4469) 2024-12-09T06:16:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T06:16:27,473 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:27,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-09T06:16:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T06:16:27,482 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:27,483 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:27,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T06:16:27,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T06:16:27,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:27,642 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T06:16:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:27,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d52f7daf429a4d14aae4863405ef3777_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724985693/Put/seqid=0 2024-12-09T06:16:27,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742013_1189 (size=12454) 2024-12-09T06:16:27,760 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#A#compaction#156 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:27,761 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/4a3ac1f5973f4666b3d79df30aa43383 is 175, key is test_row_0/A:col10/1733724985684/Put/seqid=0 2024-12-09T06:16:27,766 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/862aaae3646e4a1fb60fbea444557849 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/862aaae3646e4a1fb60fbea444557849 2024-12-09T06:16:27,779 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/B of edaa2b3dd1776fd6538daec178f1f68f into 862aaae3646e4a1fb60fbea444557849(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:27,780 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:27,780 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/B, priority=13, startTime=1733724987241; duration=0sec 2024-12-09T06:16:27,780 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:27,780 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:B 2024-12-09T06:16:27,780 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:27,782 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:27,782 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): edaa2b3dd1776fd6538daec178f1f68f/C is initiating minor compaction (all files) 2024-12-09T06:16:27,782 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of edaa2b3dd1776fd6538daec178f1f68f/C in TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:27,783 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c982aefae4ca452aa4b7ae6c4bb1d61a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/030efb05c4cd4239aaa4f93e80a9c360, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3586ac947f3d43d69ba2317a86c53414] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp, totalSize=36.2 K 2024-12-09T06:16:27,783 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c982aefae4ca452aa4b7ae6c4bb1d61a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733724982186 2024-12-09T06:16:27,784 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 030efb05c4cd4239aaa4f93e80a9c360, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733724983361 2024-12-09T06:16:27,785 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3586ac947f3d43d69ba2317a86c53414, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733724984586 2024-12-09T06:16:27,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742014_1190 (size=31801) 2024-12-09T06:16:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T06:16:27,811 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/4a3ac1f5973f4666b3d79df30aa43383 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4a3ac1f5973f4666b3d79df30aa43383 2024-12-09T06:16:27,816 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): edaa2b3dd1776fd6538daec178f1f68f#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:27,816 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/e84eba2df58148e29d99548061e9a6df is 50, key is test_row_0/C:col10/1733724985684/Put/seqid=0 2024-12-09T06:16:27,841 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/A of edaa2b3dd1776fd6538daec178f1f68f into 4a3ac1f5973f4666b3d79df30aa43383(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:27,841 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:27,841 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/A, priority=13, startTime=1733724987239; duration=0sec 2024-12-09T06:16:27,842 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:27,842 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:A 2024-12-09T06:16:27,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742015_1191 (size=12847) 2024-12-09T06:16:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:28,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724987971, max timestamp=1733724987999Num uniques 859; 2024-12-09T06:16:28,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724987971, max timestamp=1733724987999Num uniques 859; 2024-12-09T06:16:28,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:28,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:28,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733724987971, max timestamp=1733724987999Num uniques 859; 2024-12-09T06:16:28,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:28,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725048071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725048077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T06:16:28,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:28,154 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d52f7daf429a4d14aae4863405ef3777_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d52f7daf429a4d14aae4863405ef3777_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/d0b8d7b89d95437284ea3130636ae071, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:28,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/d0b8d7b89d95437284ea3130636ae071 is 175, key is test_row_0/A:col10/1733724985693/Put/seqid=0 2024-12-09T06:16:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725048187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742016_1192 (size=31255) 2024-12-09T06:16:28,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725048197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,220 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T06:16:28,271 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/e84eba2df58148e29d99548061e9a6df as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/e84eba2df58148e29d99548061e9a6df 2024-12-09T06:16:28,277 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in edaa2b3dd1776fd6538daec178f1f68f/C of edaa2b3dd1776fd6538daec178f1f68f into e84eba2df58148e29d99548061e9a6df(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:28,278 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:28,278 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., storeName=edaa2b3dd1776fd6538daec178f1f68f/C, priority=13, startTime=1733724987243; duration=0sec 2024-12-09T06:16:28,278 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:28,278 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: edaa2b3dd1776fd6538daec178f1f68f:C 2024-12-09T06:16:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725048400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725048417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,602 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/d0b8d7b89d95437284ea3130636ae071 2024-12-09T06:16:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T06:16:28,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/24933b22c6a24c40a73b04fc469404c9 is 50, key is test_row_0/B:col10/1733724985693/Put/seqid=0 2024-12-09T06:16:28,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742017_1193 (size=12301) 2024-12-09T06:16:28,672 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/24933b22c6a24c40a73b04fc469404c9 2024-12-09T06:16:28,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/39c42823ef304efda1eef7eb986fa57b is 50, key is test_row_0/C:col10/1733724985693/Put/seqid=0 2024-12-09T06:16:28,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725048721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725048726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742018_1194 (size=12301) 2024-12-09T06:16:28,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49202 deadline: 1733725048789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,794 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49254 deadline: 1733725048797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:28,806 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in edaa2b3dd1776fd6538daec178f1f68f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:29,181 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/39c42823ef304efda1eef7eb986fa57b 2024-12-09T06:16:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/d0b8d7b89d95437284ea3130636ae071 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d0b8d7b89d95437284ea3130636ae071 2024-12-09T06:16:29,242 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d0b8d7b89d95437284ea3130636ae071, entries=150, sequenceid=297, filesize=30.5 K 2024-12-09T06:16:29,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725049237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:29,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/24933b22c6a24c40a73b04fc469404c9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24933b22c6a24c40a73b04fc469404c9 2024-12-09T06:16:29,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725049247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:29,266 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24933b22c6a24c40a73b04fc469404c9, entries=150, sequenceid=297, filesize=12.0 K 2024-12-09T06:16:29,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/39c42823ef304efda1eef7eb986fa57b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/39c42823ef304efda1eef7eb986fa57b 2024-12-09T06:16:29,286 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/39c42823ef304efda1eef7eb986fa57b, entries=150, sequenceid=297, filesize=12.0 K 2024-12-09T06:16:29,287 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for edaa2b3dd1776fd6538daec178f1f68f in 1645ms, sequenceid=297, compaction requested=false 2024-12-09T06:16:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-09T06:16:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-09T06:16:29,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-09T06:16:29,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-12-09T06:16:29,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8200 sec 2024-12-09T06:16:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T06:16:29,637 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:29,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-09T06:16:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T06:16:29,645 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:29,646 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:29,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T06:16:29,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T06:16:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:29,799 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-09T06:16:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:29,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:29,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:29,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:29,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:29,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab9fa48a8c53477490df1db067bdf456_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724988050/Put/seqid=0 2024-12-09T06:16:29,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742019_1195 (size=12454) 2024-12-09T06:16:29,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:29,903 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab9fa48a8c53477490df1db067bdf456_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab9fa48a8c53477490df1db067bdf456_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:29,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/830d99e4a149459ea9cabf9ba7637909, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:29,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/830d99e4a149459ea9cabf9ba7637909 is 175, key is test_row_0/A:col10/1733724988050/Put/seqid=0 2024-12-09T06:16:29,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742020_1196 (size=31255) 2024-12-09T06:16:29,945 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/830d99e4a149459ea9cabf9ba7637909 2024-12-09T06:16:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T06:16:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/1a983c231d7842a3a94cdf130d8f3459 is 50, key is test_row_0/B:col10/1733724988050/Put/seqid=0 2024-12-09T06:16:30,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742021_1197 (size=12301) 2024-12-09T06:16:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T06:16:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:30,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:30,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:30,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725050304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725050312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:30,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:30,410 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/1a983c231d7842a3a94cdf130d8f3459 2024-12-09T06:16:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725050426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725050426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:30,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/4af7c99c0ac84c67b8e3d32575e4b557 is 50, key is test_row_0/C:col10/1733724988050/Put/seqid=0 2024-12-09T06:16:30,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742022_1198 (size=12301) 2024-12-09T06:16:30,509 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/4af7c99c0ac84c67b8e3d32575e4b557 2024-12-09T06:16:30,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/830d99e4a149459ea9cabf9ba7637909 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/830d99e4a149459ea9cabf9ba7637909 2024-12-09T06:16:30,554 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/830d99e4a149459ea9cabf9ba7637909, entries=150, sequenceid=337, filesize=30.5 K 2024-12-09T06:16:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/1a983c231d7842a3a94cdf130d8f3459 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/1a983c231d7842a3a94cdf130d8f3459 2024-12-09T06:16:30,606 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/1a983c231d7842a3a94cdf130d8f3459, entries=150, sequenceid=337, filesize=12.0 K 2024-12-09T06:16:30,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/4af7c99c0ac84c67b8e3d32575e4b557 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4af7c99c0ac84c67b8e3d32575e4b557 2024-12-09T06:16:30,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49194 deadline: 1733725050648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:30,658 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4af7c99c0ac84c67b8e3d32575e4b557, entries=150, sequenceid=337, filesize=12.0 K 2024-12-09T06:16:30,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49232 deadline: 1733725050648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:16:30,666 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for edaa2b3dd1776fd6538daec178f1f68f in 867ms, sequenceid=337, compaction requested=true 2024-12-09T06:16:30,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:30,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:30,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-09T06:16:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-09T06:16:30,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-09T06:16:30,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-12-09T06:16:30,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.0420 sec 2024-12-09T06:16:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T06:16:30,777 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:30,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-09T06:16:30,780 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:30,782 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:30,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T06:16:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T06:16:30,935 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-12-09T06:16:30,936 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:30,936 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:30,936 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,937 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-12-09T06:16:30,937 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:30,937 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,937 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T06:16:30,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:30,938 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-12-09T06:16:30,938 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:30,938 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:30,938 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,938 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:30,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:30,941 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-12-09T06:16:30,941 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:30,941 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,941 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:30,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093c24485456424159a2aaf32558e10527_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724990286/Put/seqid=0 2024-12-09T06:16:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:30,971 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-12-09T06:16:30,971 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:30,971 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,971 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:30,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. as already flushing 2024-12-09T06:16:30,972 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-12-09T06:16:30,973 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:30,973 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,973 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:30,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:30,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742023_1199 (size=12454) 2024-12-09T06:16:30,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:31,003 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093c24485456424159a2aaf32558e10527_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093c24485456424159a2aaf32558e10527_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:31,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/74571501e48741b3a9d56c2fba8f7954, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:31,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/74571501e48741b3a9d56c2fba8f7954 is 175, key is test_row_0/A:col10/1733724990286/Put/seqid=0 2024-12-09T06:16:31,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742024_1200 (size=31255) 2024-12-09T06:16:31,030 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/74571501e48741b3a9d56c2fba8f7954 2024-12-09T06:16:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T06:16:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/ab319e4be84245fdbe2e9a660591c23b is 50, key is test_row_0/B:col10/1733724990286/Put/seqid=0 2024-12-09T06:16:31,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742025_1201 (size=12301) 2024-12-09T06:16:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T06:16:31,530 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/ab319e4be84245fdbe2e9a660591c23b 2024-12-09T06:16:31,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/1a95159c127f452c9b1895f274571bc3 is 50, key is test_row_0/C:col10/1733724990286/Put/seqid=0 2024-12-09T06:16:31,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742026_1202 (size=12301) 2024-12-09T06:16:31,543 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/1a95159c127f452c9b1895f274571bc3 2024-12-09T06:16:31,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/74571501e48741b3a9d56c2fba8f7954 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/74571501e48741b3a9d56c2fba8f7954 2024-12-09T06:16:31,553 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/74571501e48741b3a9d56c2fba8f7954, entries=150, sequenceid=347, filesize=30.5 K 2024-12-09T06:16:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/ab319e4be84245fdbe2e9a660591c23b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/ab319e4be84245fdbe2e9a660591c23b 2024-12-09T06:16:31,567 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/ab319e4be84245fdbe2e9a660591c23b, entries=150, sequenceid=347, filesize=12.0 K 2024-12-09T06:16:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/1a95159c127f452c9b1895f274571bc3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1a95159c127f452c9b1895f274571bc3 2024-12-09T06:16:31,572 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1a95159c127f452c9b1895f274571bc3, entries=150, sequenceid=347, filesize=12.0 K 2024-12-09T06:16:31,572 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for edaa2b3dd1776fd6538daec178f1f68f in 634ms, sequenceid=347, compaction requested=true 2024-12-09T06:16:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for edaa2b3dd1776fd6538daec178f1f68f: 2024-12-09T06:16:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-09T06:16:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-09T06:16:31,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-09T06:16:31,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 792 msec 2024-12-09T06:16:31,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 798 msec 2024-12-09T06:16:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T06:16:31,917 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:32,817 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-12-09T06:16:32,817 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:32,817 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:32,818 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:32,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:32,828 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-12-09T06:16:32,828 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:32,828 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:32,828 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:32,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:32,898 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-12-09T06:16:32,898 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:32,898 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:32,898 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:32,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2251 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2073 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1031 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3093 rows 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1030 2024-12-09T06:16:32,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3084 rows 2024-12-09T06:16:32,899 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T06:16:32,899 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:16:32,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:32,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:32,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:32,900 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T06:16:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T06:16:32,903 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724992903"}]},"ts":"1733724992903"} 2024-12-09T06:16:32,905 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T06:16:32,905 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T06:16:32,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:16:32,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, UNASSIGN}] 2024-12-09T06:16:32,908 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, UNASSIGN 2024-12-09T06:16:32,909 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=edaa2b3dd1776fd6538daec178f1f68f, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:32,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, UNASSIGN because future has completed 2024-12-09T06:16:32,911 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:16:32,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:16:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T06:16:33,065 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:33,065 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:16:33,065 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing edaa2b3dd1776fd6538daec178f1f68f, disabling compactions & flushes 2024-12-09T06:16:33,065 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:33,065 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:33,065 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. after waiting 0 ms 2024-12-09T06:16:33,065 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:33,066 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing edaa2b3dd1776fd6538daec178f1f68f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T06:16:33,066 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=A 2024-12-09T06:16:33,066 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:33,066 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=B 2024-12-09T06:16:33,066 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:33,066 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK edaa2b3dd1776fd6538daec178f1f68f, store=C 2024-12-09T06:16:33,066 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:33,072 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209824d3fe7fd484458a34c8d33f67a0602_edaa2b3dd1776fd6538daec178f1f68f is 50, key is test_row_0/A:col10/1733724992897/Put/seqid=0 2024-12-09T06:16:33,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742027_1203 (size=12454) 2024-12-09T06:16:33,088 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:33,093 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209824d3fe7fd484458a34c8d33f67a0602_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209824d3fe7fd484458a34c8d33f67a0602_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:33,097 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/73ce546f30e24ee58bb91c090927de9a, store: [table=TestAcidGuarantees family=A region=edaa2b3dd1776fd6538daec178f1f68f] 2024-12-09T06:16:33,097 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/73ce546f30e24ee58bb91c090927de9a is 175, key is test_row_0/A:col10/1733724992897/Put/seqid=0 2024-12-09T06:16:33,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742028_1204 (size=31255) 2024-12-09T06:16:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T06:16:33,504 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/73ce546f30e24ee58bb91c090927de9a 2024-12-09T06:16:33,514 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/19b62e1cf26d411eaa665c556a160026 is 50, key is test_row_0/B:col10/1733724992897/Put/seqid=0 2024-12-09T06:16:33,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742029_1205 (size=12301) 2024-12-09T06:16:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T06:16:33,527 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/19b62e1cf26d411eaa665c556a160026 2024-12-09T06:16:33,539 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/d3f48899a1a4484eb868335ab874ec31 is 50, key is test_row_0/C:col10/1733724992897/Put/seqid=0 2024-12-09T06:16:33,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742030_1206 (size=12301) 2024-12-09T06:16:33,581 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/d3f48899a1a4484eb868335ab874ec31 2024-12-09T06:16:33,587 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/A/73ce546f30e24ee58bb91c090927de9a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/73ce546f30e24ee58bb91c090927de9a 2024-12-09T06:16:33,593 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/73ce546f30e24ee58bb91c090927de9a, entries=150, sequenceid=355, filesize=30.5 K 2024-12-09T06:16:33,595 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/B/19b62e1cf26d411eaa665c556a160026 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/19b62e1cf26d411eaa665c556a160026 2024-12-09T06:16:33,600 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/19b62e1cf26d411eaa665c556a160026, entries=150, sequenceid=355, filesize=12.0 K 2024-12-09T06:16:33,602 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/.tmp/C/d3f48899a1a4484eb868335ab874ec31 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/d3f48899a1a4484eb868335ab874ec31 2024-12-09T06:16:33,611 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/d3f48899a1a4484eb868335ab874ec31, entries=150, sequenceid=355, filesize=12.0 K 2024-12-09T06:16:33,612 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for edaa2b3dd1776fd6538daec178f1f68f in 547ms, sequenceid=355, compaction requested=true 2024-12-09T06:16:33,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/07889b634d7847d29a9a2ce26dd32c2d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/036ef63581fa4037862b828b487e5538, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/7486ef0d76774174a43aa2f3becc9ce2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4b2371938bfa4dfc95218ca909ea7abc, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/b72819f6156e4dbbb0ebacd0981430df, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/89a274368ca44de4a569dba2f5d4f24c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/eb0dcb53050e4283af0d2b2ae7cfd5f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/910747e5a78d405aa6b6dde1aa6ad850, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cb5a4badc16c40058027b1b815a3e2a1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/86ca7166bd0345dcb38953967d8a2f76, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d9ee816dd71f4bb69870ee55e3b80b6a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/293e93b1bc1b4dbc8c1ea520e6c40da9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cdc26ba3344d41a0837c5ef26a9ec6c9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a917c706a0cd4937835ae77d5591acdd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a54f17f8f2da4cf5b7afe7f43d534383, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/e4653695a93a4a38a08a0ed223773080] to archive 2024-12-09T06:16:33,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:33,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/07889b634d7847d29a9a2ce26dd32c2d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/07889b634d7847d29a9a2ce26dd32c2d 2024-12-09T06:16:33,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/036ef63581fa4037862b828b487e5538 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/036ef63581fa4037862b828b487e5538 2024-12-09T06:16:33,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/7486ef0d76774174a43aa2f3becc9ce2 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/7486ef0d76774174a43aa2f3becc9ce2 2024-12-09T06:16:33,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4b2371938bfa4dfc95218ca909ea7abc to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4b2371938bfa4dfc95218ca909ea7abc 2024-12-09T06:16:33,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/b72819f6156e4dbbb0ebacd0981430df to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/b72819f6156e4dbbb0ebacd0981430df 2024-12-09T06:16:33,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/89a274368ca44de4a569dba2f5d4f24c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/89a274368ca44de4a569dba2f5d4f24c 2024-12-09T06:16:33,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/eb0dcb53050e4283af0d2b2ae7cfd5f3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/eb0dcb53050e4283af0d2b2ae7cfd5f3 2024-12-09T06:16:33,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/910747e5a78d405aa6b6dde1aa6ad850 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/910747e5a78d405aa6b6dde1aa6ad850 2024-12-09T06:16:33,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cb5a4badc16c40058027b1b815a3e2a1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cb5a4badc16c40058027b1b815a3e2a1 2024-12-09T06:16:33,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/86ca7166bd0345dcb38953967d8a2f76 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/86ca7166bd0345dcb38953967d8a2f76 2024-12-09T06:16:33,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d9ee816dd71f4bb69870ee55e3b80b6a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d9ee816dd71f4bb69870ee55e3b80b6a 2024-12-09T06:16:33,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/293e93b1bc1b4dbc8c1ea520e6c40da9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/293e93b1bc1b4dbc8c1ea520e6c40da9 2024-12-09T06:16:33,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cdc26ba3344d41a0837c5ef26a9ec6c9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/cdc26ba3344d41a0837c5ef26a9ec6c9 2024-12-09T06:16:33,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a917c706a0cd4937835ae77d5591acdd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a917c706a0cd4937835ae77d5591acdd 2024-12-09T06:16:33,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a54f17f8f2da4cf5b7afe7f43d534383 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/a54f17f8f2da4cf5b7afe7f43d534383 2024-12-09T06:16:33,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/e4653695a93a4a38a08a0ed223773080 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/e4653695a93a4a38a08a0ed223773080 2024-12-09T06:16:33,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/4b6cffbc256547418b8be2f25589cfd9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/cf497e431e574e7987ac7349da8845c9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/80976f146ba941c99c1e8a37d8b57515, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c0c8bce224e34152a79cabd79bf27c04, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/9513429e0fe34669b147edc632eb7da1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/54d075d8104b4ee4a0dd0140acd912c9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/60cd88b1f44c44e099b40c129157909b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/7f17a7a3e5594b9aa62231ee6079025a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c836c66da4a24763a8f9c44b0e176b81, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/8b60cebe6ca7422e80bf0e2a1a6bbe3f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/99271baa31e643d582f8993d469167d8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/3e0ba2bfc6754bd4b52172b23765298c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24dce6f9d4404ccf9d1a7289ffe2abbf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/860fc8e902184b2fae433f489b697144, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/619fdfd34b834de1a1318d8b6706c992, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/aa9eb03ea3cb481985831fa7f68c9580] to archive 2024-12-09T06:16:33,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:33,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/4b6cffbc256547418b8be2f25589cfd9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/4b6cffbc256547418b8be2f25589cfd9 2024-12-09T06:16:33,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/cf497e431e574e7987ac7349da8845c9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/cf497e431e574e7987ac7349da8845c9 2024-12-09T06:16:33,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/80976f146ba941c99c1e8a37d8b57515 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/80976f146ba941c99c1e8a37d8b57515 2024-12-09T06:16:33,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c0c8bce224e34152a79cabd79bf27c04 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c0c8bce224e34152a79cabd79bf27c04 2024-12-09T06:16:33,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/9513429e0fe34669b147edc632eb7da1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/9513429e0fe34669b147edc632eb7da1 2024-12-09T06:16:33,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/54d075d8104b4ee4a0dd0140acd912c9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/54d075d8104b4ee4a0dd0140acd912c9 2024-12-09T06:16:33,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/60cd88b1f44c44e099b40c129157909b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/60cd88b1f44c44e099b40c129157909b 2024-12-09T06:16:33,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/7f17a7a3e5594b9aa62231ee6079025a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/7f17a7a3e5594b9aa62231ee6079025a 2024-12-09T06:16:33,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c836c66da4a24763a8f9c44b0e176b81 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/c836c66da4a24763a8f9c44b0e176b81 2024-12-09T06:16:33,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/8b60cebe6ca7422e80bf0e2a1a6bbe3f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/8b60cebe6ca7422e80bf0e2a1a6bbe3f 2024-12-09T06:16:33,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/99271baa31e643d582f8993d469167d8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/99271baa31e643d582f8993d469167d8 2024-12-09T06:16:33,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/3e0ba2bfc6754bd4b52172b23765298c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/3e0ba2bfc6754bd4b52172b23765298c 2024-12-09T06:16:33,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24dce6f9d4404ccf9d1a7289ffe2abbf to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24dce6f9d4404ccf9d1a7289ffe2abbf 2024-12-09T06:16:33,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/860fc8e902184b2fae433f489b697144 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/860fc8e902184b2fae433f489b697144 2024-12-09T06:16:33,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/619fdfd34b834de1a1318d8b6706c992 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/619fdfd34b834de1a1318d8b6706c992 2024-12-09T06:16:33,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/aa9eb03ea3cb481985831fa7f68c9580 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/aa9eb03ea3cb481985831fa7f68c9580 2024-12-09T06:16:33,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c8623b538b58472491f0084e801d48be, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/ea907b96c6b546959c9f61203a7e1070, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/b976f3fdb5134583a80946c2eaa1cce4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c49fc2c2cb6f4d7d8b42133174c92b6b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/160626f900fc49d084d05498c2980fb1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/f8b34b3284644c29b12b60d0940e8490, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c79d6e7f55ee438ab0b3869a7c4e64f4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/99d55289380b489e8c7e253ee0285fb8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3b2a9b8a7fcc422bb3faba1140df228b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3887fa2708544574b5e489bcbc9adec3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4421a2c5235b4d248db6891940e67d64, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1fb3967191e741f8ba6450818e8f60e7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c982aefae4ca452aa4b7ae6c4bb1d61a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/de05586dcccf4cde9fdd806626c371d4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/030efb05c4cd4239aaa4f93e80a9c360, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3586ac947f3d43d69ba2317a86c53414] to archive 2024-12-09T06:16:33,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:33,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c8623b538b58472491f0084e801d48be to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c8623b538b58472491f0084e801d48be 2024-12-09T06:16:33,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/ea907b96c6b546959c9f61203a7e1070 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/ea907b96c6b546959c9f61203a7e1070 2024-12-09T06:16:33,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/b976f3fdb5134583a80946c2eaa1cce4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/b976f3fdb5134583a80946c2eaa1cce4 2024-12-09T06:16:33,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c49fc2c2cb6f4d7d8b42133174c92b6b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c49fc2c2cb6f4d7d8b42133174c92b6b 2024-12-09T06:16:33,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/160626f900fc49d084d05498c2980fb1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/160626f900fc49d084d05498c2980fb1 2024-12-09T06:16:33,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/f8b34b3284644c29b12b60d0940e8490 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/f8b34b3284644c29b12b60d0940e8490 2024-12-09T06:16:33,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c79d6e7f55ee438ab0b3869a7c4e64f4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c79d6e7f55ee438ab0b3869a7c4e64f4 2024-12-09T06:16:33,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/99d55289380b489e8c7e253ee0285fb8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/99d55289380b489e8c7e253ee0285fb8 2024-12-09T06:16:33,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3b2a9b8a7fcc422bb3faba1140df228b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3b2a9b8a7fcc422bb3faba1140df228b 2024-12-09T06:16:33,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3887fa2708544574b5e489bcbc9adec3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3887fa2708544574b5e489bcbc9adec3 2024-12-09T06:16:33,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4421a2c5235b4d248db6891940e67d64 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4421a2c5235b4d248db6891940e67d64 2024-12-09T06:16:33,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1fb3967191e741f8ba6450818e8f60e7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1fb3967191e741f8ba6450818e8f60e7 2024-12-09T06:16:33,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c982aefae4ca452aa4b7ae6c4bb1d61a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/c982aefae4ca452aa4b7ae6c4bb1d61a 2024-12-09T06:16:33,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/de05586dcccf4cde9fdd806626c371d4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/de05586dcccf4cde9fdd806626c371d4 2024-12-09T06:16:33,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/030efb05c4cd4239aaa4f93e80a9c360 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/030efb05c4cd4239aaa4f93e80a9c360 2024-12-09T06:16:33,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3586ac947f3d43d69ba2317a86c53414 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/3586ac947f3d43d69ba2317a86c53414 2024-12-09T06:16:33,752 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/recovered.edits/358.seqid, newMaxSeqId=358, maxSeqId=4 2024-12-09T06:16:33,752 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. 2024-12-09T06:16:33,752 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for edaa2b3dd1776fd6538daec178f1f68f: Waiting for close lock at 1733724993065Running coprocessor pre-close hooks at 1733724993065Disabling compacts and flushes for region at 1733724993065Disabling writes for close at 1733724993065Obtaining lock to block concurrent updates at 1733724993066 (+1 ms)Preparing flush snapshotting stores in edaa2b3dd1776fd6538daec178f1f68f at 1733724993066Finished memstore snapshotting TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733724993066Flushing stores of TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f. at 1733724993067 (+1 ms)Flushing edaa2b3dd1776fd6538daec178f1f68f/A: creating writer at 1733724993067Flushing mob file edaa2b3dd1776fd6538daec178f1f68f/A: appending metadata at 1733724993072 (+5 ms)Flushing mob file edaa2b3dd1776fd6538daec178f1f68f/A: closing flushed file at 1733724993072Flushing edaa2b3dd1776fd6538daec178f1f68f/A: appending metadata at 1733724993097 (+25 ms)Flushing edaa2b3dd1776fd6538daec178f1f68f/A: closing flushed file at 1733724993097Flushing edaa2b3dd1776fd6538daec178f1f68f/B: creating writer at 1733724993510 (+413 ms)Flushing edaa2b3dd1776fd6538daec178f1f68f/B: appending metadata at 1733724993513 (+3 ms)Flushing edaa2b3dd1776fd6538daec178f1f68f/B: closing flushed file at 1733724993513Flushing edaa2b3dd1776fd6538daec178f1f68f/C: creating writer at 1733724993533 (+20 ms)Flushing edaa2b3dd1776fd6538daec178f1f68f/C: appending metadata at 1733724993539 (+6 ms)Flushing edaa2b3dd1776fd6538daec178f1f68f/C: closing flushed file at 1733724993539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5958429a: reopening flushed file at 1733724993586 (+47 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@213cd999: reopening flushed file at 1733724993593 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a2acd6a: reopening flushed file at 1733724993601 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for edaa2b3dd1776fd6538daec178f1f68f in 547ms, sequenceid=355, compaction requested=true at 1733724993612 (+11 ms)Writing region close event to WAL at 1733724993747 (+135 ms)Running coprocessor post-close hooks at 1733724993752 (+5 ms)Closed at 1733724993752 2024-12-09T06:16:33,755 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:33,755 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=edaa2b3dd1776fd6538daec178f1f68f, regionState=CLOSED 2024-12-09T06:16:33,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:16:33,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-09T06:16:33,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure edaa2b3dd1776fd6538daec178f1f68f, server=82ac79f0323b,42797,1733724930590 in 847 msec 2024-12-09T06:16:33,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-09T06:16:33,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=edaa2b3dd1776fd6538daec178f1f68f, UNASSIGN in 853 msec 2024-12-09T06:16:33,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-09T06:16:33,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 858 msec 2024-12-09T06:16:33,767 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724993767"}]},"ts":"1733724993767"} 2024-12-09T06:16:33,769 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T06:16:33,769 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T06:16:33,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 870 msec 2024-12-09T06:16:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T06:16:34,037 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:34,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T06:16:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:34,039 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T06:16:34,040 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:34,043 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,045 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/recovered.edits] 2024-12-09T06:16:34,048 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4a3ac1f5973f4666b3d79df30aa43383 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/4a3ac1f5973f4666b3d79df30aa43383 2024-12-09T06:16:34,049 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/73ce546f30e24ee58bb91c090927de9a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/73ce546f30e24ee58bb91c090927de9a 2024-12-09T06:16:34,051 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/74571501e48741b3a9d56c2fba8f7954 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/74571501e48741b3a9d56c2fba8f7954 2024-12-09T06:16:34,052 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/830d99e4a149459ea9cabf9ba7637909 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/830d99e4a149459ea9cabf9ba7637909 2024-12-09T06:16:34,053 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d0b8d7b89d95437284ea3130636ae071 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/A/d0b8d7b89d95437284ea3130636ae071 2024-12-09T06:16:34,056 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/19b62e1cf26d411eaa665c556a160026 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/19b62e1cf26d411eaa665c556a160026 2024-12-09T06:16:34,057 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/1a983c231d7842a3a94cdf130d8f3459 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/1a983c231d7842a3a94cdf130d8f3459 2024-12-09T06:16:34,058 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24933b22c6a24c40a73b04fc469404c9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/24933b22c6a24c40a73b04fc469404c9 2024-12-09T06:16:34,060 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/862aaae3646e4a1fb60fbea444557849 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/862aaae3646e4a1fb60fbea444557849 2024-12-09T06:16:34,061 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/ab319e4be84245fdbe2e9a660591c23b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/B/ab319e4be84245fdbe2e9a660591c23b 2024-12-09T06:16:34,063 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1a95159c127f452c9b1895f274571bc3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/1a95159c127f452c9b1895f274571bc3 2024-12-09T06:16:34,064 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/39c42823ef304efda1eef7eb986fa57b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/39c42823ef304efda1eef7eb986fa57b 2024-12-09T06:16:34,066 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4af7c99c0ac84c67b8e3d32575e4b557 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/4af7c99c0ac84c67b8e3d32575e4b557 2024-12-09T06:16:34,067 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/d3f48899a1a4484eb868335ab874ec31 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/d3f48899a1a4484eb868335ab874ec31 2024-12-09T06:16:34,068 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/e84eba2df58148e29d99548061e9a6df to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/C/e84eba2df58148e29d99548061e9a6df 2024-12-09T06:16:34,070 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/recovered.edits/358.seqid to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f/recovered.edits/358.seqid 2024-12-09T06:16:34,071 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,071 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T06:16:34,071 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T06:16:34,072 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T06:16:34,076 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091f0a1c8dd2c24e9daea30a549a18d860_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091f0a1c8dd2c24e9daea30a549a18d860_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,078 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120929a70e50e2b84f5d9af6b05162fc907f_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120929a70e50e2b84f5d9af6b05162fc907f_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,079 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092c33b7dd15624c14884611880961e6d3_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092c33b7dd15624c14884611880961e6d3_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,080 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935afada78e1042a185f146a1faaf702a_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935afada78e1042a185f146a1faaf702a_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,081 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093c24485456424159a2aaf32558e10527_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093c24485456424159a2aaf32558e10527_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,082 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096ec9e1e7cb6d482fa2a8b53535bb69aa_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096ec9e1e7cb6d482fa2a8b53535bb69aa_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,083 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979b559e64e684882913e5f9d64250572_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979b559e64e684882913e5f9d64250572_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,085 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209824d3fe7fd484458a34c8d33f67a0602_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209824d3fe7fd484458a34c8d33f67a0602_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,086 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209888e5778aa8b41feaddaf01a828d3c6d_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209888e5778aa8b41feaddaf01a828d3c6d_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,087 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab9fa48a8c53477490df1db067bdf456_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab9fa48a8c53477490df1db067bdf456_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,088 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ac227835b4b54205a5bd9e1bb7372dbc_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ac227835b4b54205a5bd9e1bb7372dbc_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,090 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3100450cab2476880a69f695eb3bf7e_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3100450cab2476880a69f695eb3bf7e_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,091 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c6213d50558b40abacc49e45d27d957b_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c6213d50558b40abacc49e45d27d957b_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,092 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cff884ccc6a84c279a1c75329572972f_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209cff884ccc6a84c279a1c75329572972f_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,094 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d52f7daf429a4d14aae4863405ef3777_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d52f7daf429a4d14aae4863405ef3777_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,095 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f27e4cae3c5e49aa83186b7b6a739510_edaa2b3dd1776fd6538daec178f1f68f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f27e4cae3c5e49aa83186b7b6a739510_edaa2b3dd1776fd6538daec178f1f68f 2024-12-09T06:16:34,096 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T06:16:34,099 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:34,105 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T06:16:34,110 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T06:16:34,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:34,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T06:16:34,112 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733724994111"}]},"ts":"9223372036854775807"} 2024-12-09T06:16:34,114 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T06:16:34,114 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => edaa2b3dd1776fd6538daec178f1f68f, NAME => 'TestAcidGuarantees,,1733724967619.edaa2b3dd1776fd6538daec178f1f68f.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T06:16:34,114 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T06:16:34,114 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733724994114"}]},"ts":"9223372036854775807"} 2024-12-09T06:16:34,118 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T06:16:34,119 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:34,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-12-09T06:16:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T06:16:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T06:16:34,151 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:34,157 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 245) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_45612531_22 at /127.0.0.1:58598 [Waiting for operation #204] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-35193471_22 at /127.0.0.1:44308 [Waiting for operation #234] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1057 (was 993) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4615 (was 3872) - AvailableMemoryMB LEAK? - 2024-12-09T06:16:34,166 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1057, ProcessCount=11, AvailableMemoryMB=4615 2024-12-09T06:16:34,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:16:34,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:16:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:34,169 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T06:16:34,169 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:34,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-09T06:16:34,170 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T06:16:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T06:16:34,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742031_1207 (size=1061) 2024-12-09T06:16:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T06:16:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T06:16:34,577 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d 2024-12-09T06:16:34,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742032_1208 (size=53) 2024-12-09T06:16:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T06:16:35,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:16:35,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f4594c92fbabea614394c10ba9359df8, disabling compactions & flushes 2024-12-09T06:16:35,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:35,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:35,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. after waiting 0 ms 2024-12-09T06:16:35,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:35,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:35,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f4594c92fbabea614394c10ba9359df8: Waiting for close lock at 1733724995013Disabling compacts and flushes for region at 1733724995013Disabling writes for close at 1733724995013Writing region close event to WAL at 1733724995013Closed at 1733724995013 2024-12-09T06:16:35,017 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T06:16:35,017 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733724995017"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733724995017"}]},"ts":"1733724995017"} 2024-12-09T06:16:35,021 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T06:16:35,025 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T06:16:35,026 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724995025"}]},"ts":"1733724995025"} 2024-12-09T06:16:35,028 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T06:16:35,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, ASSIGN}] 2024-12-09T06:16:35,029 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, ASSIGN 2024-12-09T06:16:35,032 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, ASSIGN; state=OFFLINE, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=false 2024-12-09T06:16:35,183 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f4594c92fbabea614394c10ba9359df8, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:35,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, ASSIGN because future has completed 2024-12-09T06:16:35,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:16:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T06:16:35,352 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:35,352 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:16:35,353 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,353 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:16:35,353 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,353 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,355 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,356 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:35,356 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4594c92fbabea614394c10ba9359df8 columnFamilyName A 2024-12-09T06:16:35,356 DEBUG [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:35,357 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.HStore(327): Store=f4594c92fbabea614394c10ba9359df8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:35,357 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,359 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:35,359 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4594c92fbabea614394c10ba9359df8 columnFamilyName B 2024-12-09T06:16:35,359 DEBUG [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:35,360 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.HStore(327): Store=f4594c92fbabea614394c10ba9359df8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:35,360 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,362 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:16:35,362 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4594c92fbabea614394c10ba9359df8 columnFamilyName C 2024-12-09T06:16:35,362 DEBUG [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:16:35,363 INFO [StoreOpener-f4594c92fbabea614394c10ba9359df8-1 {}] regionserver.HStore(327): Store=f4594c92fbabea614394c10ba9359df8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:16:35,363 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:35,363 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,365 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,365 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,367 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,367 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,367 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:16:35,369 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,371 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:16:35,372 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened f4594c92fbabea614394c10ba9359df8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71202708, jitterRate=0.061003029346466064}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:16:35,372 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:35,373 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for f4594c92fbabea614394c10ba9359df8: Running coprocessor pre-open hook at 1733724995353Writing region info on filesystem at 1733724995353Initializing all the Stores at 1733724995354 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724995354Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724995354Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733724995354Cleaning up temporary data from old regions at 1733724995367 (+13 ms)Running coprocessor post-open hooks at 1733724995372 (+5 ms)Region opened successfully at 1733724995373 (+1 ms) 2024-12-09T06:16:35,377 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., pid=61, masterSystemTime=1733724995349 2024-12-09T06:16:35,383 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f4594c92fbabea614394c10ba9359df8, regionState=OPEN, openSeqNum=2, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:35,383 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:35,383 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:35,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:16:35,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-09T06:16:35,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 in 193 msec 2024-12-09T06:16:35,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-09T06:16:35,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, ASSIGN in 361 msec 2024-12-09T06:16:35,394 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T06:16:35,394 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733724995394"}]},"ts":"1733724995394"} 2024-12-09T06:16:35,397 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T06:16:35,401 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T06:16:35,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2340 sec 2024-12-09T06:16:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T06:16:36,317 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:36,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,319 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f07b641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,321 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d9ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,322 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,325 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T06:16:36,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 82ac79f0323b,43241,1733724929416 2024-12-09T06:16:36,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7105195a 2024-12-09T06:16:36,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T06:16:36,327 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T06:16:36,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,330 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fabc58c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6578ea86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,335 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f34483, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,337 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76397609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,343 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d1c584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d404dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,348 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151c8cf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,349 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a70d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,358 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4c0652, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,360 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9c49c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,363 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581bed0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d384664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,370 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131f5fe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,372 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6162f927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,383 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e70b2e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,386 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1372f32b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,393 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca82c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,396 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46099d93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:16:36,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:16:36,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:16:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:16:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:16:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f2b4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:16:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:16:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:36,412 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:16:36,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35269229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:16:36,413 DEBUG [Thread-993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,414 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:36,414 DEBUG [Thread-1002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,414 DEBUG [Thread-1006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,415 DEBUG [Thread-1000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-09T06:16:36,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,419 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T06:16:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,419 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,426 DEBUG [Thread-1008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,427 DEBUG [Thread-1008 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,427 DEBUG [Thread-1008 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,429 DEBUG [Thread-991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,430 DEBUG [Thread-991 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,437 DEBUG [Thread-989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,440 DEBUG [Thread-991 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,444 DEBUG [Thread-997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:36,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:36,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:36,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:36,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:36,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:36,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:36,466 DEBUG [Thread-995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,467 DEBUG [Thread-995 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,467 DEBUG [Thread-995 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,470 DEBUG [Thread-1004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:16:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:16:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:16:36,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:16:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:16:36,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1e7bf4c100054cbdb063ea24e8ff4458 is 50, key is test_row_0/A:col10/1733724996424/Put/seqid=0 2024-12-09T06:16:36,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724996465, max timestamp=1733724996515Num uniques 859; 2024-12-09T06:16:36,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:36,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724996465, max timestamp=1733724996515Num uniques 859; 2024-12-09T06:16:36,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:36,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724996465, max timestamp=1733724996515Num uniques 859; 2024-12-09T06:16:36,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T06:16:36,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742033_1209 (size=14341) 2024-12-09T06:16:36,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1e7bf4c100054cbdb063ea24e8ff4458 2024-12-09T06:16:36,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:36,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733724996543, max timestamp=1733724996577Num uniques 1009; 2024-12-09T06:16:36,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733724996543, max timestamp=1733724996577Num uniques 1009; 2024-12-09T06:16:36,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=14558, suffix heap size=132904,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=318 2024-12-09T06:16:36,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=14558, suffix heap size=132904,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=318 2024-12-09T06:16:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733724996543, max timestamp=1733724996577Num uniques 1009; 2024-12-09T06:16:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=14558, suffix heap size=132904,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=318 2024-12-09T06:16:36,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/50b1cf764b6744f29efd362bc613575a is 50, key is test_row_0/B:col10/1733724996424/Put/seqid=0 2024-12-09T06:16:36,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725056627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725056628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725056635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725056637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725056639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742034_1210 (size=12001) 2024-12-09T06:16:36,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:36,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T06:16:36,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725056748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725056749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725056758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725056758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725056761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:36,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:36,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725056968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725056968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725056968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725056968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:36,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725056976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T06:16:37,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/50b1cf764b6744f29efd362bc613575a 2024-12-09T06:16:37,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ceb9c4a88af74e9eaa42bcac66cf76f2 is 50, key is test_row_0/C:col10/1733724996424/Put/seqid=0 2024-12-09T06:16:37,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742035_1211 (size=12001) 2024-12-09T06:16:37,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ceb9c4a88af74e9eaa42bcac66cf76f2 2024-12-09T06:16:37,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1e7bf4c100054cbdb063ea24e8ff4458 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1e7bf4c100054cbdb063ea24e8ff4458 2024-12-09T06:16:37,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:37,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:37,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1e7bf4c100054cbdb063ea24e8ff4458, entries=200, sequenceid=11, filesize=14.0 K 2024-12-09T06:16:37,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/50b1cf764b6744f29efd362bc613575a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/50b1cf764b6744f29efd362bc613575a 2024-12-09T06:16:37,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/50b1cf764b6744f29efd362bc613575a, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T06:16:37,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ceb9c4a88af74e9eaa42bcac66cf76f2 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ceb9c4a88af74e9eaa42bcac66cf76f2 2024-12-09T06:16:37,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ceb9c4a88af74e9eaa42bcac66cf76f2, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T06:16:37,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f4594c92fbabea614394c10ba9359df8 in 803ms, sequenceid=11, compaction requested=false 2024-12-09T06:16:37,262 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T06:16:37,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:37,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-12-09T06:16:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:37,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:37,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:16:37,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:37,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:16:37,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:37,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:16:37,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1cfa94e2e8ac48cb854a5badc799c54b is 50, key is test_row_0/A:col10/1733724996637/Put/seqid=0 2024-12-09T06:16:37,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725057299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725057302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725057305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725057307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742036_1212 (size=14341) 2024-12-09T06:16:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725057308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1cfa94e2e8ac48cb854a5badc799c54b 2024-12-09T06:16:37,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/35b33e6b75924fbe808a78c96b26f95f is 50, key is test_row_0/B:col10/1733724996637/Put/seqid=0 2024-12-09T06:16:37,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725057418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725057419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725057419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725057427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725057429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742037_1213 (size=12001) 2024-12-09T06:16:37,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/35b33e6b75924fbe808a78c96b26f95f 2024-12-09T06:16:37,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/36a8a91602984a2bb35fa3944eb29af7 is 50, key is test_row_0/C:col10/1733724996637/Put/seqid=0 2024-12-09T06:16:37,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742038_1214 (size=12001) 2024-12-09T06:16:37,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/36a8a91602984a2bb35fa3944eb29af7 2024-12-09T06:16:37,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1cfa94e2e8ac48cb854a5badc799c54b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cfa94e2e8ac48cb854a5badc799c54b 2024-12-09T06:16:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T06:16:37,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cfa94e2e8ac48cb854a5badc799c54b, entries=200, sequenceid=66, filesize=14.0 K 2024-12-09T06:16:37,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/35b33e6b75924fbe808a78c96b26f95f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/35b33e6b75924fbe808a78c96b26f95f 2024-12-09T06:16:37,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/35b33e6b75924fbe808a78c96b26f95f, entries=150, sequenceid=66, filesize=11.7 K 2024-12-09T06:16:37,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/36a8a91602984a2bb35fa3944eb29af7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/36a8a91602984a2bb35fa3944eb29af7 2024-12-09T06:16:37,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/36a8a91602984a2bb35fa3944eb29af7, entries=150, sequenceid=66, filesize=11.7 K 2024-12-09T06:16:37,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~450.42 KB/461232, currentSize=33.54 KB/34350 for f4594c92fbabea614394c10ba9359df8 in 291ms, sequenceid=66, compaction requested=false 2024-12-09T06:16:37,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:37,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:16:37,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:37,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:37,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:37,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:37,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:37,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:37,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/43aa22bf13134903b1e377f50a9f0318 is 50, key is test_row_0/A:col10/1733724997643/Put/seqid=0 2024-12-09T06:16:37,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742039_1215 (size=19021) 2024-12-09T06:16:37,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/43aa22bf13134903b1e377f50a9f0318 2024-12-09T06:16:37,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/0d0627c9cb4b41cc92016f3c7b75977d is 50, key is test_row_0/B:col10/1733724997643/Put/seqid=0 2024-12-09T06:16:37,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725057714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725057726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725057737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725057738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725057738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742040_1216 (size=12001) 2024-12-09T06:16:37,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/0d0627c9cb4b41cc92016f3c7b75977d 2024-12-09T06:16:37,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ec773f05edcd40b8905efe0ac7668e15 is 50, key is test_row_0/C:col10/1733724997643/Put/seqid=0 2024-12-09T06:16:37,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:37,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:37,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725057846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725057847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725057857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725057858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725057867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:37,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742041_1217 (size=12001) 2024-12-09T06:16:37,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:38,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725058057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725058057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725058077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725058077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725058077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:38,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:38,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ec773f05edcd40b8905efe0ac7668e15 2024-12-09T06:16:38,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/43aa22bf13134903b1e377f50a9f0318 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/43aa22bf13134903b1e377f50a9f0318 2024-12-09T06:16:38,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:38,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/43aa22bf13134903b1e377f50a9f0318, entries=300, sequenceid=77, filesize=18.6 K 2024-12-09T06:16:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/0d0627c9cb4b41cc92016f3c7b75977d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0d0627c9cb4b41cc92016f3c7b75977d 2024-12-09T06:16:38,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0d0627c9cb4b41cc92016f3c7b75977d, entries=150, sequenceid=77, filesize=11.7 K 2024-12-09T06:16:38,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ec773f05edcd40b8905efe0ac7668e15 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ec773f05edcd40b8905efe0ac7668e15 2024-12-09T06:16:38,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ec773f05edcd40b8905efe0ac7668e15, entries=150, sequenceid=77, filesize=11.7 K 2024-12-09T06:16:38,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f4594c92fbabea614394c10ba9359df8 in 718ms, sequenceid=77, compaction requested=true 2024-12-09T06:16:38,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:38,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:38,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:38,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:38,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:38,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:38,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-09T06:16:38,364 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:38,364 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:38,368 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:38,368 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:38,368 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,368 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ceb9c4a88af74e9eaa42bcac66cf76f2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/36a8a91602984a2bb35fa3944eb29af7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ec773f05edcd40b8905efe0ac7668e15] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=35.2 K 2024-12-09T06:16:38,369 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:38,369 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:38,369 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,369 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1e7bf4c100054cbdb063ea24e8ff4458, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cfa94e2e8ac48cb854a5badc799c54b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/43aa22bf13134903b1e377f50a9f0318] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=46.6 K 2024-12-09T06:16:38,370 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7bf4c100054cbdb063ea24e8ff4458, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733724996424 2024-12-09T06:16:38,370 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb9c4a88af74e9eaa42bcac66cf76f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733724996424 2024-12-09T06:16:38,371 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cfa94e2e8ac48cb854a5badc799c54b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733724996637 2024-12-09T06:16:38,371 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a8a91602984a2bb35fa3944eb29af7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733724996637 2024-12-09T06:16:38,371 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43aa22bf13134903b1e377f50a9f0318, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733724997299 2024-12-09T06:16:38,371 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ec773f05edcd40b8905efe0ac7668e15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733724997299 2024-12-09T06:16:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:38,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=140.89 KB heapSize=419.81 KB 2024-12-09T06:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:38,407 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:38,408 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/3a038e2f29314551810d0a0c34b0107e is 50, key is test_row_0/A:col10/1733724997643/Put/seqid=0 2024-12-09T06:16:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725058395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725058397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725058399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/0aaf00549b6d4e67bedc8d90befedadd is 50, key is test_row_0/A:col10/1733724997726/Put/seqid=0 2024-12-09T06:16:38,427 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:38,428 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/317cc2f36b8948c7acb159fd0680d789 is 50, key is test_row_0/C:col10/1733724997643/Put/seqid=0 2024-12-09T06:16:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725058415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725058422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:38,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742042_1218 (size=12104) 2024-12-09T06:16:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742043_1219 (size=12001) 2024-12-09T06:16:38,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/0aaf00549b6d4e67bedc8d90befedadd 2024-12-09T06:16:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725058527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725058527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725058537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742044_1220 (size=12104) 2024-12-09T06:16:38,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725058549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/92fefa3305584dc4932d0a8575d8ed92 is 50, key is test_row_0/B:col10/1733724997726/Put/seqid=0 2024-12-09T06:16:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T06:16:38,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742045_1221 (size=12001) 2024-12-09T06:16:38,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/92fefa3305584dc4932d0a8575d8ed92 2024-12-09T06:16:38,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/7b3da2442ee7471090735258991afa6b is 50, key is test_row_0/C:col10/1733724997726/Put/seqid=0 2024-12-09T06:16:38,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742046_1222 (size=12001) 2024-12-09T06:16:38,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/7b3da2442ee7471090735258991afa6b 2024-12-09T06:16:38,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/0aaf00549b6d4e67bedc8d90befedadd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/0aaf00549b6d4e67bedc8d90befedadd 2024-12-09T06:16:38,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/0aaf00549b6d4e67bedc8d90befedadd, entries=150, sequenceid=101, filesize=11.7 K 2024-12-09T06:16:38,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/92fefa3305584dc4932d0a8575d8ed92 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/92fefa3305584dc4932d0a8575d8ed92 2024-12-09T06:16:38,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/92fefa3305584dc4932d0a8575d8ed92, entries=150, sequenceid=101, filesize=11.7 K 2024-12-09T06:16:38,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/7b3da2442ee7471090735258991afa6b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/7b3da2442ee7471090735258991afa6b 2024-12-09T06:16:38,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/7b3da2442ee7471090735258991afa6b, entries=150, sequenceid=101, filesize=11.7 K 2024-12-09T06:16:38,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~419.77 KB/429840, currentSize=33.54 KB/34350 for f4594c92fbabea614394c10ba9359df8 in 362ms, sequenceid=101, compaction requested=true 2024-12-09T06:16:38,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:38,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:38,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:38,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:38,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-09T06:16:38,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:38,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-09T06:16:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:38,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:38,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:38,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:38,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:38,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:38,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:38,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:38,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/fa633d9dfeae48238fdeb47243bb61e1 is 50, key is test_row_0/A:col10/1733724998754/Put/seqid=0 2024-12-09T06:16:38,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:38,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742047_1223 (size=16681) 2024-12-09T06:16:38,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/fa633d9dfeae48238fdeb47243bb61e1 2024-12-09T06:16:38,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724998830, max timestamp=1733724998850Num uniques 859; 2024-12-09T06:16:38,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:38,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724998830, max timestamp=1733724998850Num uniques 859; 2024-12-09T06:16:38,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:38,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724998830, max timestamp=1733724998850Num uniques 859; 2024-12-09T06:16:38,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:38,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/0af493373c1742aba685277c195d46c5 is 50, key is test_row_0/B:col10/1733724998754/Put/seqid=0 2024-12-09T06:16:38,906 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/3a038e2f29314551810d0a0c34b0107e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3a038e2f29314551810d0a0c34b0107e 2024-12-09T06:16:38,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742048_1224 (size=12001) 2024-12-09T06:16:38,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/0af493373c1742aba685277c195d46c5 2024-12-09T06:16:38,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725058918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725058918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725058921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,933 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into 3a038e2f29314551810d0a0c34b0107e(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:38,933 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,933 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=13, startTime=1733724998364; duration=0sec 2024-12-09T06:16:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,933 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-09T06:16:38,933 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:38,933 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:38,933 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-09T06:16:38,934 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:16:38,935 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:16:38,935 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. because compaction request was cancelled 2024-12-09T06:16:38,935 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:38,935 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:38,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725058926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,939 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:38,939 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:38,939 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,940 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/50b1cf764b6744f29efd362bc613575a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/35b33e6b75924fbe808a78c96b26f95f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0d0627c9cb4b41cc92016f3c7b75977d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/92fefa3305584dc4932d0a8575d8ed92] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=46.9 K 2024-12-09T06:16:38,940 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b1cf764b6744f29efd362bc613575a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733724996424 2024-12-09T06:16:38,941 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35b33e6b75924fbe808a78c96b26f95f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733724996637 2024-12-09T06:16:38,942 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0627c9cb4b41cc92016f3c7b75977d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733724997299 2024-12-09T06:16:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725058931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:38,949 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92fefa3305584dc4932d0a8575d8ed92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733724997726 2024-12-09T06:16:38,954 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/317cc2f36b8948c7acb159fd0680d789 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/317cc2f36b8948c7acb159fd0680d789 2024-12-09T06:16:38,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3c542f0191a146a098a954f6507d65a9 is 50, key is test_row_0/C:col10/1733724998754/Put/seqid=0 2024-12-09T06:16:38,970 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into 317cc2f36b8948c7acb159fd0680d789(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:38,970 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=13, startTime=1733724998364; duration=0sec 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. because compaction request was cancelled 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:38,970 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:16:38,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:38,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:38,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:38,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:38,978 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:16:38,979 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:16:38,979 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. because compaction request was cancelled 2024-12-09T06:16:38,979 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:38,983 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:38,984 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/d59e98b38a06441893997d036cda005c is 50, key is test_row_0/B:col10/1733724997726/Put/seqid=0 2024-12-09T06:16:39,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742049_1225 (size=12001) 2024-12-09T06:16:39,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3c542f0191a146a098a954f6507d65a9 2024-12-09T06:16:39,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/fa633d9dfeae48238fdeb47243bb61e1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fa633d9dfeae48238fdeb47243bb61e1 2024-12-09T06:16:39,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fa633d9dfeae48238fdeb47243bb61e1, entries=250, sequenceid=111, filesize=16.3 K 2024-12-09T06:16:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/0af493373c1742aba685277c195d46c5 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0af493373c1742aba685277c195d46c5 2024-12-09T06:16:39,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725059036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:39,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:39,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:39,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:39,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725059037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:39,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725059037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:39,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0af493373c1742aba685277c195d46c5, entries=150, sequenceid=111, filesize=11.7 K 2024-12-09T06:16:39,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3c542f0191a146a098a954f6507d65a9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3c542f0191a146a098a954f6507d65a9 2024-12-09T06:16:39,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742050_1226 (size=12139) 2024-12-09T06:16:39,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3c542f0191a146a098a954f6507d65a9, entries=150, sequenceid=111, filesize=11.7 K 2024-12-09T06:16:39,073 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/d59e98b38a06441893997d036cda005c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d59e98b38a06441893997d036cda005c 2024-12-09T06:16:39,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=132.99 KB/136185 for f4594c92fbabea614394c10ba9359df8 in 318ms, sequenceid=111, compaction requested=true 2024-12-09T06:16:39,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:39,081 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:39,083 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:39,083 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:39,083 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,083 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3a038e2f29314551810d0a0c34b0107e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/0aaf00549b6d4e67bedc8d90befedadd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fa633d9dfeae48238fdeb47243bb61e1] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=39.8 K 2024-12-09T06:16:39,083 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a038e2f29314551810d0a0c34b0107e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733724997299 2024-12-09T06:16:39,084 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aaf00549b6d4e67bedc8d90befedadd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733724997726 2024-12-09T06:16:39,084 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting fa633d9dfeae48238fdeb47243bb61e1, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733724998391 2024-12-09T06:16:39,085 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into d59e98b38a06441893997d036cda005c(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:39,085 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:39,085 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=12, startTime=1733724998741; duration=0sec 2024-12-09T06:16:39,085 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:39,085 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:39,085 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:39,085 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:39,090 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:39,090 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:39,090 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,090 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/317cc2f36b8948c7acb159fd0680d789, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/7b3da2442ee7471090735258991afa6b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3c542f0191a146a098a954f6507d65a9] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=35.3 K 2024-12-09T06:16:39,092 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317cc2f36b8948c7acb159fd0680d789, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1733724997299 2024-12-09T06:16:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:39,093 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b3da2442ee7471090735258991afa6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733724997726 2024-12-09T06:16:39,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724998912, max timestamp=1733724999086Num uniques 1009; 2024-12-09T06:16:39,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724998912, max timestamp=1733724999086Num uniques 1009; 2024-12-09T06:16:39,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T06:16:39,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T06:16:39,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724998912, max timestamp=1733724999086Num uniques 1009; 2024-12-09T06:16:39,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T06:16:39,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-12-09T06:16:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:39,100 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c542f0191a146a098a954f6507d65a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733724998398 2024-12-09T06:16:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:39,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:39,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/24a6328b9a8749a98cc557adaf0b9410 is 50, key is test_row_0/A:col10/1733724998912/Put/seqid=0 2024-12-09T06:16:39,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:39,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:39,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,144 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:39,145 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/5465170d0d014985b09eba45552d7bfa is 50, key is test_row_0/A:col10/1733724998754/Put/seqid=0 2024-12-09T06:16:39,172 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#190 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:39,172 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/d3696616279f442ea4492cc99fdb84df is 50, key is test_row_0/C:col10/1733724998754/Put/seqid=0 2024-12-09T06:16:39,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742051_1227 (size=14541) 2024-12-09T06:16:39,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742052_1228 (size=12207) 2024-12-09T06:16:39,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742053_1229 (size=12207) 2024-12-09T06:16:39,265 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/d3696616279f442ea4492cc99fdb84df as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/d3696616279f442ea4492cc99fdb84df 2024-12-09T06:16:39,283 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into d3696616279f442ea4492cc99fdb84df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:39,283 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:39,283 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=13, startTime=1733724999082; duration=0sec 2024-12-09T06:16:39,285 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:39,285 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:39,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724999247, max timestamp=1733724999282Num uniques 859; 2024-12-09T06:16:39,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:39,286 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:16:39,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724999247, max timestamp=1733724999282Num uniques 859; 2024-12-09T06:16:39,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:39,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,289 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:16:39,289 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:16:39,289 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. because compaction request was cancelled 2024-12-09T06:16:39,289 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724999269, max timestamp=1733724999282Num uniques 859; 2024-12-09T06:16:39,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:16:39,309 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T06:16:39,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724999309, max timestamp=1733724999361Num uniques 1009; 2024-12-09T06:16:39,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724999309, max timestamp=1733724999361Num uniques 1009; 2024-12-09T06:16:39,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T06:16:39,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T06:16:39,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733724999309, max timestamp=1733724999361Num uniques 1009; 2024-12-09T06:16:39,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T06:16:39,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733724999420, max timestamp=1733724999439Num uniques 1027; 2024-12-09T06:16:39,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T06:16:39,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733724999420, max timestamp=1733724999439Num uniques 1027; 2024-12-09T06:16:39,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T06:16:39,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733724999420, max timestamp=1733724999439Num uniques 1027; 2024-12-09T06:16:39,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T06:16:39,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733724999474, max timestamp=1733724999489Num uniques 1036; 2024-12-09T06:16:39,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T06:16:39,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733724999474, max timestamp=1733724999489Num uniques 1036; 2024-12-09T06:16:39,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T06:16:39,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733724999474, max timestamp=1733724999489Num uniques 1036; 2024-12-09T06:16:39,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T06:16:39,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733724999539, max timestamp=1733724999554Num uniques 1045; 2024-12-09T06:16:39,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-09T06:16:39,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733724999539, max timestamp=1733724999554Num uniques 1045; 2024-12-09T06:16:39,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-09T06:16:39,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733724999539, max timestamp=1733724999554Num uniques 1045; 2024-12-09T06:16:39,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-09T06:16:39,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/24a6328b9a8749a98cc557adaf0b9410 2024-12-09T06:16:39,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,652 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/5465170d0d014985b09eba45552d7bfa as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/5465170d0d014985b09eba45552d7bfa 2024-12-09T06:16:39,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733724999628, max timestamp=1733724999648Num uniques 1104; 2024-12-09T06:16:39,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-09T06:16:39,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733724999628, max timestamp=1733724999648Num uniques 1104; 2024-12-09T06:16:39,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-09T06:16:39,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724999630, max timestamp=1733724999648Num uniques 1104; 2024-12-09T06:16:39,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-12-09T06:16:39,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/a2c0733b7c3442218a4aa84e0e41c4c3 is 50, key is test_row_0/B:col10/1733724998912/Put/seqid=0 2024-12-09T06:16:39,694 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into 5465170d0d014985b09eba45552d7bfa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:39,694 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:39,695 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=13, startTime=1733724999081; duration=0sec 2024-12-09T06:16:39,695 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:39,695 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:39,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742054_1230 (size=12151) 2024-12-09T06:16:39,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733724999694, max timestamp=1733724999749Num uniques 1063; 2024-12-09T06:16:39,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T06:16:39,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733724999694, max timestamp=1733724999749Num uniques 1063; 2024-12-09T06:16:39,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T06:16:39,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733724999694, max timestamp=1733724999749Num uniques 1009; 2024-12-09T06:16:39,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=9748, suffix heap size=132904,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=213 2024-12-09T06:16:39,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733724999809, max timestamp=1733724999822Num uniques 1072; 2024-12-09T06:16:39,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-09T06:16:39,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733724999809, max timestamp=1733724999822Num uniques 1072; 2024-12-09T06:16:39,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-09T06:16:39,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733724999809, max timestamp=1733724999822Num uniques 1072; 2024-12-09T06:16:39,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-09T06:16:39,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733724999866, max timestamp=1733724999890Num uniques 1081; 2024-12-09T06:16:39,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724999866, max timestamp=1733724999890Num uniques 1081; 2024-12-09T06:16:39,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-09T06:16:39,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=6870, suffix heap size=140392,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=150 2024-12-09T06:16:39,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733724999866, max timestamp=1733724999890Num uniques 1081; 2024-12-09T06:16:39,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-09T06:16:39,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:39,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:39,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8695, totalHeapSize=20000, min timestamp=1733724999930, max timestamp=1733724999942Num uniques 1040; 2024-12-09T06:16:39,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47625, new segment data size=8695, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-09T06:16:39,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733724999930, max timestamp=1733724999942Num uniques 1040; 2024-12-09T06:16:39,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-09T06:16:39,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724999930, max timestamp=1733724999942Num uniques 1009; 2024-12-09T06:16:39,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46204, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-09T06:16:39,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733724999970, max timestamp=1733724999984Num uniques 1049; 2024-12-09T06:16:39,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-09T06:16:39,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733724999970, max timestamp=1733724999984Num uniques 1049; 2024-12-09T06:16:39,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733724999970, max timestamp=1733724999984Num uniques 1009; 2024-12-09T06:16:39,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-09T06:16:39,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-09T06:16:40,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725060033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725060034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733724999984, max timestamp=1733725000037Num uniques 1108; 2024-12-09T06:16:40,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-12-09T06:16:40,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733724999984, max timestamp=1733725000037Num uniques 1108; 2024-12-09T06:16:40,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-12-09T06:16:40,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733724999984, max timestamp=1733725000037Num uniques 1009; 2024-12-09T06:16:40,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46206, new segment data size=7230, suffix heap size=132904,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=158 2024-12-09T06:16:40,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:40,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/a2c0733b7c3442218a4aa84e0e41c4c3 2024-12-09T06:16:40,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733725000100, max timestamp=1733725000119Num uniques 1017; 2024-12-09T06:16:40,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733725000100, max timestamp=1733725000119Num uniques 1017; 2024-12-09T06:16:40,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T06:16:40,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T06:16:40,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733725000100, max timestamp=1733725000119Num uniques 1017; 2024-12-09T06:16:40,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T06:16:40,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/6c13df52b7d0473997a34459228f5a84 is 50, key is test_row_0/C:col10/1733724998912/Put/seqid=0 2024-12-09T06:16:40,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733725000161, max timestamp=1733725000172Num uniques 1076; 2024-12-09T06:16:40,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-09T06:16:40,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733725000161, max timestamp=1733725000172Num uniques 1076; 2024-12-09T06:16:40,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-09T06:16:40,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733725000161, max timestamp=1733725000172Num uniques 1076; 2024-12-09T06:16:40,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-09T06:16:40,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742055_1231 (size=12151) 2024-12-09T06:16:40,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10750, totalHeapSize=24680, min timestamp=1733725000198, max timestamp=1733725000223Num uniques 1035; 2024-12-09T06:16:40,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47390, new segment data size=10750, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-09T06:16:40,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733725000198, max timestamp=1733725000223Num uniques 1035; 2024-12-09T06:16:40,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-12-09T06:16:40,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733725000198, max timestamp=1733725000223Num uniques 1035; 2024-12-09T06:16:40,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-12-09T06:16:40,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:40,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:40,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733725000198, max timestamp=1733725000269Num uniques 1094; 2024-12-09T06:16:40,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-12-09T06:16:40,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733725000198, max timestamp=1733725000269Num uniques 1044; 2024-12-09T06:16:40,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-09T06:16:40,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733725000198, max timestamp=1733725000269Num uniques 1044; 2024-12-09T06:16:40,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-09T06:16:40,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733725000294, max timestamp=1733725000309Num uniques 1053; 2024-12-09T06:16:40,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-12-09T06:16:40,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725000300, max timestamp=1733725000309Num uniques 1053; 2024-12-09T06:16:40,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=6870, suffix heap size=137480,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=150 2024-12-09T06:16:40,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725000300, max timestamp=1733725000309Num uniques 1053; 2024-12-09T06:16:40,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=6870, suffix heap size=137480,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=150 2024-12-09T06:16:40,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:40,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:40,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733725000364, max timestamp=1733725000409Num uniques 1062; 2024-12-09T06:16:40,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-09T06:16:40,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733725000364, max timestamp=1733725000409Num uniques 1009; 2024-12-09T06:16:40,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46207, new segment data size=9702, suffix heap size=132904,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=212 2024-12-09T06:16:40,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733725000364, max timestamp=1733725000409Num uniques 1009; 2024-12-09T06:16:40,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46207, new segment data size=9702, suffix heap size=132904,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=212 2024-12-09T06:16:40,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733725000441, max timestamp=1733725000455Num uniques 1071; 2024-12-09T06:16:40,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-09T06:16:40,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733725000441, max timestamp=1733725000455Num uniques 1071; 2024-12-09T06:16:40,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-09T06:16:40,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733725000441, max timestamp=1733725000455Num uniques 1071; 2024-12-09T06:16:40,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=7826, suffix heap size=139352,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=171 2024-12-09T06:16:40,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1733725000500, max timestamp=1733725000508Num uniques 1030; 2024-12-09T06:16:40,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=10530, suffix heap size=135088,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=230 2024-12-09T06:16:40,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1733725000500, max timestamp=1733725000508Num uniques 1030; 2024-12-09T06:16:40,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=10530, suffix heap size=135088,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=230 2024-12-09T06:16:40,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725000500, max timestamp=1733725000508Num uniques 1030; 2024-12-09T06:16:40,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=6870, suffix heap size=135088,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=150 2024-12-09T06:16:40,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/6c13df52b7d0473997a34459228f5a84 2024-12-09T06:16:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T06:16:40,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725000531, max timestamp=1733725000549Num uniques 1089; 2024-12-09T06:16:40,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725000531, max timestamp=1733725000549Num uniques 1089; 2024-12-09T06:16:40,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725000531, max timestamp=1733725000549Num uniques 1009; 2024-12-09T06:16:40,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=6870, suffix heap size=141224,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=150 2024-12-09T06:16:40,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=6870, suffix heap size=141224,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=150 2024-12-09T06:16:40,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-09T06:16:40,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/24a6328b9a8749a98cc557adaf0b9410 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/24a6328b9a8749a98cc557adaf0b9410 2024-12-09T06:16:40,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/24a6328b9a8749a98cc557adaf0b9410, entries=200, sequenceid=152, filesize=14.2 K 2024-12-09T06:16:40,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/a2c0733b7c3442218a4aa84e0e41c4c3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2c0733b7c3442218a4aa84e0e41c4c3 2024-12-09T06:16:40,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2c0733b7c3442218a4aa84e0e41c4c3, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T06:16:40,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/6c13df52b7d0473997a34459228f5a84 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6c13df52b7d0473997a34459228f5a84 2024-12-09T06:16:40,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1733725000541, max timestamp=1733725000657Num uniques 1009; 2024-12-09T06:16:40,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13648, suffix heap size=132904,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=298 2024-12-09T06:16:40,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=248, cellSize=11358, totalHeapSize=26032, min timestamp=1733725000639, max timestamp=1733725000657Num uniques 1009; 2024-12-09T06:16:40,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11358, suffix heap size=132904,new segment heap size=26032 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=248 2024-12-09T06:16:40,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725000639, max timestamp=1733725000657Num uniques 1009; 2024-12-09T06:16:40,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-09T06:16:40,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6c13df52b7d0473997a34459228f5a84, entries=150, sequenceid=152, filesize=11.9 K 2024-12-09T06:16:40,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=78.36 KB/80242 for f4594c92fbabea614394c10ba9359df8 in 1587ms, sequenceid=152, compaction requested=true 2024-12-09T06:16:40,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:40,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:40,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:40,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:40,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:40,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:40,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-09T06:16:40,687 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:40,688 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:16:40,692 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:40,692 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:40,692 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=85.07 KB heapSize=232.31 KB 2024-12-09T06:16:40,692 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d59e98b38a06441893997d036cda005c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0af493373c1742aba685277c195d46c5, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2c0733b7c3442218a4aa84e0e41c4c3] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=35.4 K 2024-12-09T06:16:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:40,693 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting d59e98b38a06441893997d036cda005c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733724997726 2024-12-09T06:16:40,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:40,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:40,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:40,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:40,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:40,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:40,694 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af493373c1742aba685277c195d46c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733724998398 2024-12-09T06:16:40,694 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c0733b7c3442218a4aa84e0e41c4c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733724998912 2024-12-09T06:16:40,696 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:16:40,697 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:16:40,697 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. because compaction request was cancelled 2024-12-09T06:16:40,697 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:40,697 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:16:40,704 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:16:40,704 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:16:40,704 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. because compaction request was cancelled 2024-12-09T06:16:40,704 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:40,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1ea915f89f5843ae9468d3295740aa51 is 50, key is test_row_0/A:col10/1733725000692/Put/seqid=0 2024-12-09T06:16:40,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725060727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725060730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725060731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725060732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725060732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:40,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:40,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,759 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:40,759 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/a2ef9e49ae1a464ca2a72a5ea305644f is 50, key is test_row_0/B:col10/1733724998912/Put/seqid=0 2024-12-09T06:16:40,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742056_1232 (size=12301) 2024-12-09T06:16:40,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742057_1233 (size=12391) 2024-12-09T06:16:40,850 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/a2ef9e49ae1a464ca2a72a5ea305644f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2ef9e49ae1a464ca2a72a5ea305644f 2024-12-09T06:16:40,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725060847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725060847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725060847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725060847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725060847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:40,857 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into a2ef9e49ae1a464ca2a72a5ea305644f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:40,857 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:40,857 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=13, startTime=1733725000687; duration=0sec 2024-12-09T06:16:40,857 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:40,857 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:40,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:40,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:40,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:40,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725061056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725061057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725061057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725061059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725061061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:41,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:41,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1ea915f89f5843ae9468d3295740aa51 2024-12-09T06:16:41,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/bd2e692c06e44466967689761ad8bf6e is 50, key is test_row_0/B:col10/1733725000692/Put/seqid=0 2024-12-09T06:16:41,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742058_1234 (size=12301) 2024-12-09T06:16:41,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725061367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725061367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725061367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725061367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725061368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:41,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:41,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:41,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/bd2e692c06e44466967689761ad8bf6e 2024-12-09T06:16:41,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3141fa6cc4954499a2b1365dce26c585 is 50, key is test_row_0/C:col10/1733725000692/Put/seqid=0 2024-12-09T06:16:41,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742059_1235 (size=12301) 2024-12-09T06:16:41,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:41,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:41,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:41,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:41,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725061876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725061876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725061877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725061877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725061878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:42,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:42,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:42,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:42,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:42,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:42,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:42,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.93 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3141fa6cc4954499a2b1365dce26c585 2024-12-09T06:16:42,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1ea915f89f5843ae9468d3295740aa51 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1ea915f89f5843ae9468d3295740aa51 2024-12-09T06:16:42,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1ea915f89f5843ae9468d3295740aa51, entries=150, sequenceid=544, filesize=12.0 K 2024-12-09T06:16:42,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/bd2e692c06e44466967689761ad8bf6e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/bd2e692c06e44466967689761ad8bf6e 2024-12-09T06:16:42,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/bd2e692c06e44466967689761ad8bf6e, entries=150, sequenceid=544, filesize=12.0 K 2024-12-09T06:16:42,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3141fa6cc4954499a2b1365dce26c585 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3141fa6cc4954499a2b1365dce26c585 2024-12-09T06:16:42,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3141fa6cc4954499a2b1365dce26c585, entries=150, sequenceid=544, filesize=12.0 K 2024-12-09T06:16:42,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.78 KB/93982, heapSize ~252.19 KB/258240, currentSize=93.93 KB/96180 for f4594c92fbabea614394c10ba9359df8 in 1469ms, sequenceid=544, compaction requested=true 2024-12-09T06:16:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:42,161 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:42,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:42,161 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:16:42,162 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:16:42,162 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:16:42,162 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:42,162 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. because compaction request was cancelled 2024-12-09T06:16:42,162 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:42,162 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:42,162 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:42,162 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:42,162 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/5465170d0d014985b09eba45552d7bfa, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/24a6328b9a8749a98cc557adaf0b9410, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1ea915f89f5843ae9468d3295740aa51] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=38.1 K 2024-12-09T06:16:42,163 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5465170d0d014985b09eba45552d7bfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733724998398 2024-12-09T06:16:42,163 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:42,163 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:42,164 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:42,164 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/d3696616279f442ea4492cc99fdb84df, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6c13df52b7d0473997a34459228f5a84, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3141fa6cc4954499a2b1365dce26c585] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=35.8 K 2024-12-09T06:16:42,166 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24a6328b9a8749a98cc557adaf0b9410, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733724998912 2024-12-09T06:16:42,166 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting d3696616279f442ea4492cc99fdb84df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733724998398 2024-12-09T06:16:42,167 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea915f89f5843ae9468d3295740aa51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733725000680 2024-12-09T06:16:42,167 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c13df52b7d0473997a34459228f5a84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733724998912 2024-12-09T06:16:42,168 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3141fa6cc4954499a2b1365dce26c585, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733725000680 2024-12-09T06:16:42,185 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:42,185 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/7ce77e76bf2745fab1728dc7bec1ca38 is 50, key is test_row_0/A:col10/1733725000692/Put/seqid=0 2024-12-09T06:16:42,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T06:16:42,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:42,188 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T06:16:42,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:42,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:42,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:42,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:42,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:42,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:42,194 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:42,195 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/528e7b3b327f49b9b6190bc8c15c5202 is 50, key is test_row_0/C:col10/1733725000692/Put/seqid=0 2024-12-09T06:16:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/217bc51a9a114cf5be4d411074a732ab is 50, key is test_row_0/A:col10/1733725000713/Put/seqid=0 2024-12-09T06:16:42,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742061_1237 (size=12609) 2024-12-09T06:16:42,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742060_1236 (size=12609) 2024-12-09T06:16:42,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742062_1238 (size=12301) 2024-12-09T06:16:42,706 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/217bc51a9a114cf5be4d411074a732ab 2024-12-09T06:16:42,716 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/7ce77e76bf2745fab1728dc7bec1ca38 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/7ce77e76bf2745fab1728dc7bec1ca38 2024-12-09T06:16:42,716 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/528e7b3b327f49b9b6190bc8c15c5202 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/528e7b3b327f49b9b6190bc8c15c5202 2024-12-09T06:16:42,763 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into 528e7b3b327f49b9b6190bc8c15c5202(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:42,763 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:42,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/2f1a9ba73ff1465face023dac361cae0 is 50, key is test_row_0/B:col10/1733725000713/Put/seqid=0 2024-12-09T06:16:42,763 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=13, startTime=1733725002161; duration=0sec 2024-12-09T06:16:42,765 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:42,765 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:42,767 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into 7ce77e76bf2745fab1728dc7bec1ca38(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:42,767 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:42,767 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=13, startTime=1733725002161; duration=0sec 2024-12-09T06:16:42,767 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:42,767 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:42,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742063_1239 (size=12301) 2024-12-09T06:16:42,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:42,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725062924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:42,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725062931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:42,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725062934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725062942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:42,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725062942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725063047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725063049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725063049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725063049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725063057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,205 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/2f1a9ba73ff1465face023dac361cae0 2024-12-09T06:16:43,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/02b0193a18f34ee1930420a920518bf9 is 50, key is test_row_0/C:col10/1733725000713/Put/seqid=0 2024-12-09T06:16:43,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742064_1240 (size=12301) 2024-12-09T06:16:43,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725063256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725063257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725063260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,268 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/02b0193a18f34ee1930420a920518bf9 2024-12-09T06:16:43,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725063267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725063269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/217bc51a9a114cf5be4d411074a732ab as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/217bc51a9a114cf5be4d411074a732ab 2024-12-09T06:16:43,287 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/217bc51a9a114cf5be4d411074a732ab, entries=150, sequenceid=562, filesize=12.0 K 2024-12-09T06:16:43,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/2f1a9ba73ff1465face023dac361cae0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2f1a9ba73ff1465face023dac361cae0 2024-12-09T06:16:43,295 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2f1a9ba73ff1465face023dac361cae0, entries=150, sequenceid=562, filesize=12.0 K 2024-12-09T06:16:43,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/02b0193a18f34ee1930420a920518bf9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/02b0193a18f34ee1930420a920518bf9 2024-12-09T06:16:43,308 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/02b0193a18f34ee1930420a920518bf9, entries=150, sequenceid=562, filesize=12.0 K 2024-12-09T06:16:43,309 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f4594c92fbabea614394c10ba9359df8 in 1121ms, sequenceid=562, compaction requested=true 2024-12-09T06:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-09T06:16:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-09T06:16:43,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-09T06:16:43,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 6.8940 sec 2024-12-09T06:16:43,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 6.9110 sec 2024-12-09T06:16:43,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T06:16:43,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:43,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:43,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:43,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:43,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:43,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:43,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/c5a90b54a15d4e24b0cdf9e86f6b0026 is 50, key is test_row_0/A:col10/1733725003567/Put/seqid=0 2024-12-09T06:16:43,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725063598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725063599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725063601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725063601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725063604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742065_1241 (size=12301) 2024-12-09T06:16:43,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/c5a90b54a15d4e24b0cdf9e86f6b0026 2024-12-09T06:16:43,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/fa7670b1b2bc44e6b006d45d38d69efe is 50, key is test_row_0/B:col10/1733725003567/Put/seqid=0 2024-12-09T06:16:43,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742066_1242 (size=12301) 2024-12-09T06:16:43,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/fa7670b1b2bc44e6b006d45d38d69efe 2024-12-09T06:16:43,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725063707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/2d73706c211b46fe8b7adfcb3c7f3781 is 50, key is test_row_0/C:col10/1733725003567/Put/seqid=0 2024-12-09T06:16:43,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725063725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725063729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725063729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725063729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742067_1243 (size=12301) 2024-12-09T06:16:43,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725063917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725063940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725063941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725063941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:43,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725063941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/2d73706c211b46fe8b7adfcb3c7f3781 2024-12-09T06:16:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/c5a90b54a15d4e24b0cdf9e86f6b0026 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c5a90b54a15d4e24b0cdf9e86f6b0026 2024-12-09T06:16:44,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725064227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c5a90b54a15d4e24b0cdf9e86f6b0026, entries=150, sequenceid=580, filesize=12.0 K 2024-12-09T06:16:44,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/fa7670b1b2bc44e6b006d45d38d69efe as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/fa7670b1b2bc44e6b006d45d38d69efe 2024-12-09T06:16:44,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725064247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725064247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725064257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725064258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/fa7670b1b2bc44e6b006d45d38d69efe, entries=150, sequenceid=580, filesize=12.0 K 2024-12-09T06:16:44,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/2d73706c211b46fe8b7adfcb3c7f3781 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2d73706c211b46fe8b7adfcb3c7f3781 2024-12-09T06:16:44,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2d73706c211b46fe8b7adfcb3c7f3781, entries=150, sequenceid=580, filesize=12.0 K 2024-12-09T06:16:44,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f4594c92fbabea614394c10ba9359df8 in 764ms, sequenceid=580, compaction requested=true 2024-12-09T06:16:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-09T06:16:44,334 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:44,337 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:44,342 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:44,342 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:44,342 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:44,342 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2ef9e49ae1a464ca2a72a5ea305644f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/bd2e692c06e44466967689761ad8bf6e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2f1a9ba73ff1465face023dac361cae0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/fa7670b1b2bc44e6b006d45d38d69efe] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=48.1 K 2024-12-09T06:16:44,345 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2ef9e49ae1a464ca2a72a5ea305644f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733724998912 2024-12-09T06:16:44,346 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:44,346 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:44,346 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:44,346 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/528e7b3b327f49b9b6190bc8c15c5202, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/02b0193a18f34ee1930420a920518bf9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2d73706c211b46fe8b7adfcb3c7f3781] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=36.3 K 2024-12-09T06:16:44,347 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 528e7b3b327f49b9b6190bc8c15c5202, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733725000680 2024-12-09T06:16:44,347 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd2e692c06e44466967689761ad8bf6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733725000680 2024-12-09T06:16:44,349 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b0193a18f34ee1930420a920518bf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733725000713 2024-12-09T06:16:44,349 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f1a9ba73ff1465face023dac361cae0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733725000713 2024-12-09T06:16:44,349 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d73706c211b46fe8b7adfcb3c7f3781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733725002929 2024-12-09T06:16:44,351 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7670b1b2bc44e6b006d45d38d69efe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733725002929 2024-12-09T06:16:44,379 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:44,381 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/f9c1977527814cafa3ee3f40132ad5ff is 50, key is test_row_0/C:col10/1733725003567/Put/seqid=0 2024-12-09T06:16:44,382 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:44,383 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f1fc9cc9fa2548b6a8277060aad8ce4f is 50, key is test_row_0/B:col10/1733725003567/Put/seqid=0 2024-12-09T06:16:44,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742069_1245 (size=12677) 2024-12-09T06:16:44,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742068_1244 (size=12711) 2024-12-09T06:16:44,468 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/f9c1977527814cafa3ee3f40132ad5ff as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/f9c1977527814cafa3ee3f40132ad5ff 2024-12-09T06:16:44,497 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into f9c1977527814cafa3ee3f40132ad5ff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:44,497 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:44,497 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=13, startTime=1733725004333; duration=0sec 2024-12-09T06:16:44,497 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:44,497 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:44,497 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:44,498 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:44,498 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:44,499 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:44,499 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/7ce77e76bf2745fab1728dc7bec1ca38, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/217bc51a9a114cf5be4d411074a732ab, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c5a90b54a15d4e24b0cdf9e86f6b0026] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=36.3 K 2024-12-09T06:16:44,499 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce77e76bf2745fab1728dc7bec1ca38, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733725000680 2024-12-09T06:16:44,500 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 217bc51a9a114cf5be4d411074a732ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733725000713 2024-12-09T06:16:44,503 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c5a90b54a15d4e24b0cdf9e86f6b0026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733725002929 2024-12-09T06:16:44,523 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:44,523 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/550979bb384e428c9c831663c32f2164 is 50, key is test_row_0/A:col10/1733725003567/Put/seqid=0 2024-12-09T06:16:44,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742070_1246 (size=12711) 2024-12-09T06:16:44,587 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/550979bb384e428c9c831663c32f2164 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/550979bb384e428c9c831663c32f2164 2024-12-09T06:16:44,604 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into 550979bb384e428c9c831663c32f2164(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:44,604 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:44,604 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=13, startTime=1733725004333; duration=0sec 2024-12-09T06:16:44,604 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:44,604 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T06:16:44,617 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:44,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-09T06:16:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T06:16:44,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:44,623 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:44,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T06:16:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:44,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T06:16:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:44,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/e1673c33d38a4616bb9116d2731bef94 is 50, key is test_row_0/A:col10/1733725003599/Put/seqid=0 2024-12-09T06:16:44,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742071_1247 (size=14741) 2024-12-09T06:16:44,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/e1673c33d38a4616bb9116d2731bef94 2024-12-09T06:16:44,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T06:16:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:44,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:44,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/2cd35bb49b1041ad962ec4bcbec50a25 is 50, key is test_row_0/B:col10/1733725003599/Put/seqid=0 2024-12-09T06:16:44,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725064788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725064789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725064790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725064795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725064795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742072_1248 (size=12301) 2024-12-09T06:16:44,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/2cd35bb49b1041ad962ec4bcbec50a25 2024-12-09T06:16:44,820 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f1fc9cc9fa2548b6a8277060aad8ce4f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f1fc9cc9fa2548b6a8277060aad8ce4f 2024-12-09T06:16:44,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/1e25e54d9bea42a1a2b82466c8d1abd7 is 50, key is test_row_0/C:col10/1733725003599/Put/seqid=0 2024-12-09T06:16:44,829 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into f1fc9cc9fa2548b6a8277060aad8ce4f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:44,829 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:44,829 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=12, startTime=1733725004333; duration=0sec 2024-12-09T06:16:44,829 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:44,829 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:44,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742073_1249 (size=12301) 2024-12-09T06:16:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725064906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725064907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725064907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725064907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725064907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:44,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T06:16:44,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:44,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:44,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:44,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:44,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T06:16:45,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T06:16:45,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:45,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:45,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:45,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:45,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:45,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725065117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725065117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725065117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725065118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725065118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T06:16:45,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:45,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:45,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:45,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:45,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:45,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/1e25e54d9bea42a1a2b82466c8d1abd7 2024-12-09T06:16:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T06:16:45,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/e1673c33d38a4616bb9116d2731bef94 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e1673c33d38a4616bb9116d2731bef94 2024-12-09T06:16:45,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e1673c33d38a4616bb9116d2731bef94, entries=200, sequenceid=599, filesize=14.4 K 2024-12-09T06:16:45,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/2cd35bb49b1041ad962ec4bcbec50a25 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2cd35bb49b1041ad962ec4bcbec50a25 2024-12-09T06:16:45,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2cd35bb49b1041ad962ec4bcbec50a25, entries=150, sequenceid=599, filesize=12.0 K 2024-12-09T06:16:45,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/1e25e54d9bea42a1a2b82466c8d1abd7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/1e25e54d9bea42a1a2b82466c8d1abd7 2024-12-09T06:16:45,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/1e25e54d9bea42a1a2b82466c8d1abd7, entries=150, sequenceid=599, filesize=12.0 K 2024-12-09T06:16:45,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f4594c92fbabea614394c10ba9359df8 in 536ms, sequenceid=599, compaction requested=false 2024-12-09T06:16:45,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:45,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T06:16:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:45,394 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T06:16:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:45,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4ffd3c8b0ce04ac88f126fc8e28f79ab is 50, key is test_row_0/A:col10/1733725004788/Put/seqid=0 2024-12-09T06:16:45,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742074_1250 (size=12301) 2024-12-09T06:16:45,419 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4ffd3c8b0ce04ac88f126fc8e28f79ab 2024-12-09T06:16:45,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/c45fb704708241d9a89fa81fe4e73150 is 50, key is test_row_0/B:col10/1733725004788/Put/seqid=0 2024-12-09T06:16:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742075_1251 (size=12301) 2024-12-09T06:16:45,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725065441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725065443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725065445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725065446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725065446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725065557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725065557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725065557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725065557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725065557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725065767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725065767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725065768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725065768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725065769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T06:16:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:45,845 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/c45fb704708241d9a89fa81fe4e73150 2024-12-09T06:16:45,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1 is 50, key is test_row_0/C:col10/1733725004788/Put/seqid=0 2024-12-09T06:16:45,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742076_1252 (size=12301) 2024-12-09T06:16:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725066076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725066076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725066076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725066077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725066077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,265 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1 2024-12-09T06:16:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4ffd3c8b0ce04ac88f126fc8e28f79ab as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4ffd3c8b0ce04ac88f126fc8e28f79ab 2024-12-09T06:16:46,278 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4ffd3c8b0ce04ac88f126fc8e28f79ab, entries=150, sequenceid=615, filesize=12.0 K 2024-12-09T06:16:46,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/c45fb704708241d9a89fa81fe4e73150 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/c45fb704708241d9a89fa81fe4e73150 2024-12-09T06:16:46,284 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/c45fb704708241d9a89fa81fe4e73150, entries=150, sequenceid=615, filesize=12.0 K 2024-12-09T06:16:46,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1 2024-12-09T06:16:46,315 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1, entries=150, sequenceid=615, filesize=12.0 K 2024-12-09T06:16:46,317 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f4594c92fbabea614394c10ba9359df8 in 923ms, sequenceid=615, compaction requested=true 2024-12-09T06:16:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:46,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-09T06:16:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-09T06:16:46,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-09T06:16:46,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6960 sec 2024-12-09T06:16:46,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.7050 sec 2024-12-09T06:16:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:46,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T06:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:46,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/f117b7d3a3954791b547f7dbbc962612 is 50, key is test_row_0/A:col10/1733725005444/Put/seqid=0 2024-12-09T06:16:46,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725066613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742077_1253 (size=12301) 2024-12-09T06:16:46,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725066620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725066622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725066623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725066625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725066749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725066749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725066750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725066753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725066753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T06:16:46,787 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:46,787 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-09T06:16:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T06:16:46,790 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:46,791 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:46,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T06:16:46,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T06:16:46,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:46,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:46,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:46,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725066966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725066967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725066967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725066967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725066967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/f117b7d3a3954791b547f7dbbc962612 2024-12-09T06:16:47,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/51fd74161d2a471c984fe25c3b6028f7 is 50, key is test_row_0/B:col10/1733725005444/Put/seqid=0 2024-12-09T06:16:47,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742078_1254 (size=12301) 2024-12-09T06:16:47,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T06:16:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T06:16:47,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T06:16:47,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:47,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725067276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725067276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725067277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725067277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725067277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T06:16:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T06:16:47,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/51fd74161d2a471c984fe25c3b6028f7 2024-12-09T06:16:47,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ffc26f5647d540149481cf0e8499fd18 is 50, key is test_row_0/C:col10/1733725005444/Put/seqid=0 2024-12-09T06:16:47,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742079_1255 (size=12301) 2024-12-09T06:16:47,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T06:16:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T06:16:47,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:47,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725067787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725067787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725067787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725067788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725067789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:47,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T06:16:47,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:47,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:47,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ffc26f5647d540149481cf0e8499fd18 2024-12-09T06:16:47,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/f117b7d3a3954791b547f7dbbc962612 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/f117b7d3a3954791b547f7dbbc962612 2024-12-09T06:16:47,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/f117b7d3a3954791b547f7dbbc962612, entries=150, sequenceid=634, filesize=12.0 K 2024-12-09T06:16:47,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/51fd74161d2a471c984fe25c3b6028f7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/51fd74161d2a471c984fe25c3b6028f7 2024-12-09T06:16:47,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/51fd74161d2a471c984fe25c3b6028f7, entries=150, sequenceid=634, filesize=12.0 K 2024-12-09T06:16:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/ffc26f5647d540149481cf0e8499fd18 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ffc26f5647d540149481cf0e8499fd18 2024-12-09T06:16:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T06:16:47,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ffc26f5647d540149481cf0e8499fd18, entries=150, sequenceid=634, filesize=12.0 K 2024-12-09T06:16:47,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f4594c92fbabea614394c10ba9359df8 in 1354ms, sequenceid=634, compaction requested=true 2024-12-09T06:16:47,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:47,946 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:47,946 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:47,948 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:47,948 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:47,948 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,948 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/550979bb384e428c9c831663c32f2164, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e1673c33d38a4616bb9116d2731bef94, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4ffd3c8b0ce04ac88f126fc8e28f79ab, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/f117b7d3a3954791b547f7dbbc962612] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=50.8 K 2024-12-09T06:16:47,948 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:47,949 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:47,949 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:47,949 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f1fc9cc9fa2548b6a8277060aad8ce4f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2cd35bb49b1041ad962ec4bcbec50a25, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/c45fb704708241d9a89fa81fe4e73150, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/51fd74161d2a471c984fe25c3b6028f7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=48.4 K 2024-12-09T06:16:47,949 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 550979bb384e428c9c831663c32f2164, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733725002929 2024-12-09T06:16:47,949 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f1fc9cc9fa2548b6a8277060aad8ce4f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733725002929 2024-12-09T06:16:47,950 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1673c33d38a4616bb9116d2731bef94, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733725003592 2024-12-09T06:16:47,950 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd35bb49b1041ad962ec4bcbec50a25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733725003599 2024-12-09T06:16:47,950 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffd3c8b0ce04ac88f126fc8e28f79ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733725004772 2024-12-09T06:16:47,951 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c45fb704708241d9a89fa81fe4e73150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733725004772 2024-12-09T06:16:47,951 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting f117b7d3a3954791b547f7dbbc962612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733725005444 2024-12-09T06:16:47,951 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 51fd74161d2a471c984fe25c3b6028f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733725005444 2024-12-09T06:16:47,984 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#217 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:47,985 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/69c306cb6cd84507b84fbbf0cbe1191b is 50, key is test_row_0/A:col10/1733725005444/Put/seqid=0 2024-12-09T06:16:47,995 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:47,996 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/d26559939e2c42d2ae2121c799991a3d is 50, key is test_row_0/B:col10/1733725005444/Put/seqid=0 2024-12-09T06:16:48,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T06:16:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:48,023 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T06:16:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:48,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742080_1256 (size=12847) 2024-12-09T06:16:48,042 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/69c306cb6cd84507b84fbbf0cbe1191b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/69c306cb6cd84507b84fbbf0cbe1191b 2024-12-09T06:16:48,051 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into 69c306cb6cd84507b84fbbf0cbe1191b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:48,051 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:48,051 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=12, startTime=1733725007945; duration=0sec 2024-12-09T06:16:48,051 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:48,051 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:48,052 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:48,053 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:48,053 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:48,053 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:48,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/fee2ca0afd644ffea63ea5a06f1bfff4 is 50, key is test_row_0/A:col10/1733725006613/Put/seqid=0 2024-12-09T06:16:48,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742081_1257 (size=12813) 2024-12-09T06:16:48,054 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/f9c1977527814cafa3ee3f40132ad5ff, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/1e25e54d9bea42a1a2b82466c8d1abd7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ffc26f5647d540149481cf0e8499fd18] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=48.5 K 2024-12-09T06:16:48,054 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9c1977527814cafa3ee3f40132ad5ff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733725002929 2024-12-09T06:16:48,055 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e25e54d9bea42a1a2b82466c8d1abd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1733725003599 2024-12-09T06:16:48,055 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3ab6ac5aaa4dd6a82a154b0eca6ee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733725004772 2024-12-09T06:16:48,056 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc26f5647d540149481cf0e8499fd18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733725005444 2024-12-09T06:16:48,067 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:48,068 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/a2a4fa75b7d443adb240ea215f655e85 is 50, key is test_row_0/C:col10/1733725005444/Put/seqid=0 2024-12-09T06:16:48,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742082_1258 (size=12301) 2024-12-09T06:16:48,083 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/fee2ca0afd644ffea63ea5a06f1bfff4 2024-12-09T06:16:48,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/6b1c97801562424281b40d1f14bbe905 is 50, key is test_row_0/B:col10/1733725006613/Put/seqid=0 2024-12-09T06:16:48,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742083_1259 (size=12847) 2024-12-09T06:16:48,145 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/a2a4fa75b7d443adb240ea215f655e85 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/a2a4fa75b7d443adb240ea215f655e85 2024-12-09T06:16:48,152 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into a2a4fa75b7d443adb240ea215f655e85(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:48,152 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:48,152 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=12, startTime=1733725007946; duration=0sec 2024-12-09T06:16:48,152 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:48,152 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:48,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742084_1260 (size=12301) 2024-12-09T06:16:48,173 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/6b1c97801562424281b40d1f14bbe905 2024-12-09T06:16:48,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/aaa28e139ebc40e28ebd597bea8ea2f4 is 50, key is test_row_0/C:col10/1733725006613/Put/seqid=0 2024-12-09T06:16:48,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742085_1261 (size=12301) 2024-12-09T06:16:48,207 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/aaa28e139ebc40e28ebd597bea8ea2f4 2024-12-09T06:16:48,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/fee2ca0afd644ffea63ea5a06f1bfff4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fee2ca0afd644ffea63ea5a06f1bfff4 2024-12-09T06:16:48,218 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fee2ca0afd644ffea63ea5a06f1bfff4, entries=150, sequenceid=647, filesize=12.0 K 2024-12-09T06:16:48,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/6b1c97801562424281b40d1f14bbe905 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/6b1c97801562424281b40d1f14bbe905 2024-12-09T06:16:48,226 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/6b1c97801562424281b40d1f14bbe905, entries=150, sequenceid=647, filesize=12.0 K 2024-12-09T06:16:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/aaa28e139ebc40e28ebd597bea8ea2f4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/aaa28e139ebc40e28ebd597bea8ea2f4 2024-12-09T06:16:48,233 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/aaa28e139ebc40e28ebd597bea8ea2f4, entries=150, sequenceid=647, filesize=12.0 K 2024-12-09T06:16:48,237 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for f4594c92fbabea614394c10ba9359df8 in 213ms, sequenceid=647, compaction requested=false 2024-12-09T06:16:48,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:48,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:48,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-09T06:16:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-09T06:16:48,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-09T06:16:48,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-12-09T06:16:48,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4550 sec 2024-12-09T06:16:48,465 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/d26559939e2c42d2ae2121c799991a3d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d26559939e2c42d2ae2121c799991a3d 2024-12-09T06:16:48,489 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into d26559939e2c42d2ae2121c799991a3d(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:48,489 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:48,491 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=12, startTime=1733725007946; duration=0sec 2024-12-09T06:16:48,491 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:48,491 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:48,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:48,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:48,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/2789f6127a8644c29c1f8c25e6b60b3f is 50, key is test_row_0/A:col10/1733725008797/Put/seqid=0 2024-12-09T06:16:48,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725008832, max timestamp=1733725008836Num uniques 859; 2024-12-09T06:16:48,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725008832, max timestamp=1733725008836Num uniques 859; 2024-12-09T06:16:48,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:48,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:48,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725008832, max timestamp=1733725008836Num uniques 859; 2024-12-09T06:16:48,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:48,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742086_1262 (size=17181) 2024-12-09T06:16:48,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725068864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:48,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725068866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:48,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725008858, max timestamp=1733725008871Num uniques 1018; 2024-12-09T06:16:48,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725008858, max timestamp=1733725008871Num uniques 1018; 2024-12-09T06:16:48,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:16:48,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:16:48,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725008858, max timestamp=1733725008871Num uniques 1018; 2024-12-09T06:16:48,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:16:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725068908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725068912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725068913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T06:16:48,937 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:48,937 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-09T06:16:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T06:16:48,940 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:48,940 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:48,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:48,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725068977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725068977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725069016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725069026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725069026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T06:16:49,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T06:16:49,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:49,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:49,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:49,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:49,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:49,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725069187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725069187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725069227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T06:16:49,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/2789f6127a8644c29c1f8c25e6b60b3f 2024-12-09T06:16:49,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:49,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:49,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:49,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:49,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:49,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725069252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725069252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T06:16:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/112d93884aab49ab84e16553aee521fc is 50, key is test_row_0/B:col10/1733725008797/Put/seqid=0 2024-12-09T06:16:49,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742087_1263 (size=12301) 2024-12-09T06:16:49,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/112d93884aab49ab84e16553aee521fc 2024-12-09T06:16:49,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/04e065434d954f62867b95df54fc628f is 50, key is test_row_0/C:col10/1733725008797/Put/seqid=0 2024-12-09T06:16:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742088_1264 (size=12301) 2024-12-09T06:16:49,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/04e065434d954f62867b95df54fc628f 2024-12-09T06:16:49,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/2789f6127a8644c29c1f8c25e6b60b3f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/2789f6127a8644c29c1f8c25e6b60b3f 2024-12-09T06:16:49,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/2789f6127a8644c29c1f8c25e6b60b3f, entries=250, sequenceid=660, filesize=16.8 K 2024-12-09T06:16:49,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/112d93884aab49ab84e16553aee521fc as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/112d93884aab49ab84e16553aee521fc 2024-12-09T06:16:49,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/112d93884aab49ab84e16553aee521fc, entries=150, sequenceid=660, filesize=12.0 K 2024-12-09T06:16:49,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/04e065434d954f62867b95df54fc628f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/04e065434d954f62867b95df54fc628f 2024-12-09T06:16:49,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/04e065434d954f62867b95df54fc628f, entries=150, sequenceid=660, filesize=12.0 K 2024-12-09T06:16:49,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f4594c92fbabea614394c10ba9359df8 in 555ms, sequenceid=660, compaction requested=true 2024-12-09T06:16:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:49,365 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:49,365 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:49,367 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:49,367 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:49,367 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:49,367 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:49,367 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:49,367 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:49,368 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/69c306cb6cd84507b84fbbf0cbe1191b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fee2ca0afd644ffea63ea5a06f1bfff4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/2789f6127a8644c29c1f8c25e6b60b3f] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=41.3 K 2024-12-09T06:16:49,368 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d26559939e2c42d2ae2121c799991a3d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/6b1c97801562424281b40d1f14bbe905, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/112d93884aab49ab84e16553aee521fc] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=36.5 K 2024-12-09T06:16:49,368 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting d26559939e2c42d2ae2121c799991a3d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733725005444 2024-12-09T06:16:49,368 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c306cb6cd84507b84fbbf0cbe1191b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733725005444 2024-12-09T06:16:49,369 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b1c97801562424281b40d1f14bbe905, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1733725006598 2024-12-09T06:16:49,369 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 112d93884aab49ab84e16553aee521fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733725008797 2024-12-09T06:16:49,369 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting fee2ca0afd644ffea63ea5a06f1bfff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1733725006598 2024-12-09T06:16:49,370 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2789f6127a8644c29c1f8c25e6b60b3f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733725008797 2024-12-09T06:16:49,391 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:49,392 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/70eaa425a31c47acb6ec414f95d72dec is 50, key is test_row_0/B:col10/1733725008797/Put/seqid=0 2024-12-09T06:16:49,399 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:49,399 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/d877a8afc9fe49abae2e0e0f613133cf is 50, key is test_row_0/A:col10/1733725008797/Put/seqid=0 2024-12-09T06:16:49,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T06:16:49,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:49,406 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-12-09T06:16:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:49,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/d1e21b3ebb204c7ba6f3b7346044aab6 is 50, key is test_row_0/A:col10/1733725008911/Put/seqid=0 2024-12-09T06:16:49,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742089_1265 (size=12915) 2024-12-09T06:16:49,468 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/70eaa425a31c47acb6ec414f95d72dec as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/70eaa425a31c47acb6ec414f95d72dec 2024-12-09T06:16:49,477 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into 70eaa425a31c47acb6ec414f95d72dec(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:49,477 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:49,477 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=13, startTime=1733725009365; duration=0sec 2024-12-09T06:16:49,477 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:49,477 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:49,478 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:49,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742090_1266 (size=12949) 2024-12-09T06:16:49,480 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:49,480 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:49,480 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:49,480 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/a2a4fa75b7d443adb240ea215f655e85, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/aaa28e139ebc40e28ebd597bea8ea2f4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/04e065434d954f62867b95df54fc628f] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=36.6 K 2024-12-09T06:16:49,480 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a2a4fa75b7d443adb240ea215f655e85, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1733725005444 2024-12-09T06:16:49,481 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting aaa28e139ebc40e28ebd597bea8ea2f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1733725006598 2024-12-09T06:16:49,481 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e065434d954f62867b95df54fc628f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733725008797 2024-12-09T06:16:49,485 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/d877a8afc9fe49abae2e0e0f613133cf as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d877a8afc9fe49abae2e0e0f613133cf 2024-12-09T06:16:49,494 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into d877a8afc9fe49abae2e0e0f613133cf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:49,494 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:49,496 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=13, startTime=1733725009365; duration=0sec 2024-12-09T06:16:49,496 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:49,496 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:49,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742091_1267 (size=12301) 2024-12-09T06:16:49,505 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/d1e21b3ebb204c7ba6f3b7346044aab6 2024-12-09T06:16:49,507 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:49,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:49,508 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/25881fb9a86f4dafa8f55df705fa1903 is 50, key is test_row_0/C:col10/1733725008797/Put/seqid=0 2024-12-09T06:16:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f9affa624cb0411195b04c294aa95750 is 50, key is test_row_0/B:col10/1733725008911/Put/seqid=0 2024-12-09T06:16:49,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742092_1268 (size=12949) 2024-12-09T06:16:49,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725069533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725069534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725069536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,545 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/25881fb9a86f4dafa8f55df705fa1903 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/25881fb9a86f4dafa8f55df705fa1903 2024-12-09T06:16:49,552 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into 25881fb9a86f4dafa8f55df705fa1903(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:49,552 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:49,552 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=13, startTime=1733725009365; duration=0sec 2024-12-09T06:16:49,552 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:49,552 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T06:16:49,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725069567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725069567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742093_1269 (size=12301) 2024-12-09T06:16:49,570 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f9affa624cb0411195b04c294aa95750 2024-12-09T06:16:49,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3555497bca594654aa34ace9cc5321c3 is 50, key is test_row_0/C:col10/1733725008911/Put/seqid=0 2024-12-09T06:16:49,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742094_1270 (size=12301) 2024-12-09T06:16:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725069647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725069647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725069856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725069856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:49,999 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3555497bca594654aa34ace9cc5321c3 2024-12-09T06:16:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/d1e21b3ebb204c7ba6f3b7346044aab6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d1e21b3ebb204c7ba6f3b7346044aab6 2024-12-09T06:16:50,008 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d1e21b3ebb204c7ba6f3b7346044aab6, entries=150, sequenceid=714, filesize=12.0 K 2024-12-09T06:16:50,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f9affa624cb0411195b04c294aa95750 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f9affa624cb0411195b04c294aa95750 2024-12-09T06:16:50,013 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f9affa624cb0411195b04c294aa95750, entries=150, sequenceid=714, filesize=12.0 K 2024-12-09T06:16:50,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/3555497bca594654aa34ace9cc5321c3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3555497bca594654aa34ace9cc5321c3 2024-12-09T06:16:50,017 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3555497bca594654aa34ace9cc5321c3, entries=150, sequenceid=714, filesize=12.0 K 2024-12-09T06:16:50,018 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for f4594c92fbabea614394c10ba9359df8 in 612ms, sequenceid=714, compaction requested=false 2024-12-09T06:16:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-09T06:16:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-09T06:16:50,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-09T06:16:50,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0790 sec 2024-12-09T06:16:50,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0840 sec 2024-12-09T06:16:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:50,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:50,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/30cc4d7fa9f94b4f90783c86333c4ba0 is 50, key is test_row_0/A:col10/1733725009514/Put/seqid=0 2024-12-09T06:16:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T06:16:50,077 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:50,077 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-09T06:16:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T06:16:50,080 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:50,081 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:50,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:50,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742095_1271 (size=9857) 2024-12-09T06:16:50,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725010094, max timestamp=1733725010107Num uniques 859; 2024-12-09T06:16:50,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725010094, max timestamp=1733725010107Num uniques 859; 2024-12-09T06:16:50,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:50,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:16:50,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725010094, max timestamp=1733725010107Num uniques 859; 2024-12-09T06:16:50,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725070132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725010129, max timestamp=1733725010135Num uniques 1018; 2024-12-09T06:16:50,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:16:50,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725010129, max timestamp=1733725010135Num uniques 1018; 2024-12-09T06:16:50,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:16:50,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725010129, max timestamp=1733725010135Num uniques 1068; 2024-12-09T06:16:50,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T06:16:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725070166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725070166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725070170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725070170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T06:16:50,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T06:16:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725070246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725070277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725070277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T06:16:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T06:16:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725070456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725070487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725070487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/30cc4d7fa9f94b4f90783c86333c4ba0 2024-12-09T06:16:50,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/ca19e0094e9f46bda16b28c09a6e4948 is 50, key is test_row_0/B:col10/1733725009514/Put/seqid=0 2024-12-09T06:16:50,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742096_1272 (size=9857) 2024-12-09T06:16:50,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T06:16:50,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:50,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725070676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725070676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T06:16:50,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:50,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T06:16:50,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725070766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725070796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725070796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:50,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T06:16:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:50,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/ca19e0094e9f46bda16b28c09a6e4948 2024-12-09T06:16:50,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/5e485af0897e415aa7af3c4d0f285bc7 is 50, key is test_row_0/C:col10/1733725009514/Put/seqid=0 2024-12-09T06:16:50,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742097_1273 (size=9857) 2024-12-09T06:16:50,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/5e485af0897e415aa7af3c4d0f285bc7 2024-12-09T06:16:50,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/30cc4d7fa9f94b4f90783c86333c4ba0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/30cc4d7fa9f94b4f90783c86333c4ba0 2024-12-09T06:16:50,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/30cc4d7fa9f94b4f90783c86333c4ba0, entries=100, sequenceid=727, filesize=9.6 K 2024-12-09T06:16:50,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/ca19e0094e9f46bda16b28c09a6e4948 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ca19e0094e9f46bda16b28c09a6e4948 2024-12-09T06:16:50,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ca19e0094e9f46bda16b28c09a6e4948, entries=100, sequenceid=727, filesize=9.6 K 2024-12-09T06:16:50,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/5e485af0897e415aa7af3c4d0f285bc7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/5e485af0897e415aa7af3c4d0f285bc7 2024-12-09T06:16:50,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/5e485af0897e415aa7af3c4d0f285bc7, entries=100, sequenceid=727, filesize=9.6 K 2024-12-09T06:16:50,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f4594c92fbabea614394c10ba9359df8 in 900ms, sequenceid=727, compaction requested=true 2024-12-09T06:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:50,947 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:50,947 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:50,948 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:50,948 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:50,948 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:50,948 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:50,948 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,948 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,948 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d877a8afc9fe49abae2e0e0f613133cf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d1e21b3ebb204c7ba6f3b7346044aab6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/30cc4d7fa9f94b4f90783c86333c4ba0] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=34.3 K 2024-12-09T06:16:50,948 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/70eaa425a31c47acb6ec414f95d72dec, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f9affa624cb0411195b04c294aa95750, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ca19e0094e9f46bda16b28c09a6e4948] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=34.3 K 2024-12-09T06:16:50,949 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting d877a8afc9fe49abae2e0e0f613133cf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733725008797 2024-12-09T06:16:50,949 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 70eaa425a31c47acb6ec414f95d72dec, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733725008797 2024-12-09T06:16:50,949 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f9affa624cb0411195b04c294aa95750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733725008904 2024-12-09T06:16:50,949 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e21b3ebb204c7ba6f3b7346044aab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733725008904 2024-12-09T06:16:50,949 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30cc4d7fa9f94b4f90783c86333c4ba0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733725009514 2024-12-09T06:16:50,949 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ca19e0094e9f46bda16b28c09a6e4948, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733725009514 2024-12-09T06:16:50,956 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:50,957 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/e7ce5177d7f5480eb15ac1adebc0e71d is 50, key is test_row_0/A:col10/1733725009514/Put/seqid=0 2024-12-09T06:16:50,958 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:50,959 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/7179a23fb5bf42fbb43249c992ad2a3a is 50, key is test_row_0/B:col10/1733725009514/Put/seqid=0 2024-12-09T06:16:50,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742099_1275 (size=13017) 2024-12-09T06:16:50,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742098_1274 (size=13051) 2024-12-09T06:16:50,975 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/e7ce5177d7f5480eb15ac1adebc0e71d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e7ce5177d7f5480eb15ac1adebc0e71d 2024-12-09T06:16:50,979 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into e7ce5177d7f5480eb15ac1adebc0e71d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:50,979 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:50,979 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=13, startTime=1733725010947; duration=0sec 2024-12-09T06:16:50,979 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:50,979 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:50,979 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:50,980 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:50,980 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:50,980 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,980 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/25881fb9a86f4dafa8f55df705fa1903, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3555497bca594654aa34ace9cc5321c3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/5e485af0897e415aa7af3c4d0f285bc7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=34.3 K 2024-12-09T06:16:50,980 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25881fb9a86f4dafa8f55df705fa1903, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=660, earliestPutTs=1733725008797 2024-12-09T06:16:50,980 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3555497bca594654aa34ace9cc5321c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733725008904 2024-12-09T06:16:50,981 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e485af0897e415aa7af3c4d0f285bc7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733725009514 2024-12-09T06:16:50,990 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:50,990 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/92a51bef401f4cc8ba8365b5b94edea1 is 50, key is test_row_0/C:col10/1733725009514/Put/seqid=0 2024-12-09T06:16:50,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742100_1276 (size=13051) 2024-12-09T06:16:50,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T06:16:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:50,997 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-12-09T06:16:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:51,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/c1737a1779b94e119970465e316e281a is 50, key is test_row_0/A:col10/1733725010161/Put/seqid=0 2024-12-09T06:16:51,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742101_1277 (size=12301) 2024-12-09T06:16:51,005 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=781 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/c1737a1779b94e119970465e316e281a 2024-12-09T06:16:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/9bb28673f8be4424b378b22e60239251 is 50, key is test_row_0/B:col10/1733725010161/Put/seqid=0 2024-12-09T06:16:51,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742102_1278 (size=12301) 2024-12-09T06:16:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T06:16:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:51,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:51,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725071303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725071306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725071306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,376 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/7179a23fb5bf42fbb43249c992ad2a3a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7179a23fb5bf42fbb43249c992ad2a3a 2024-12-09T06:16:51,382 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into 7179a23fb5bf42fbb43249c992ad2a3a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:51,382 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:51,382 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=13, startTime=1733725010947; duration=0sec 2024-12-09T06:16:51,382 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:51,382 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:51,400 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/92a51bef401f4cc8ba8365b5b94edea1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/92a51bef401f4cc8ba8365b5b94edea1 2024-12-09T06:16:51,406 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into 92a51bef401f4cc8ba8365b5b94edea1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:51,406 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:51,406 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=13, startTime=1733725010947; duration=0sec 2024-12-09T06:16:51,406 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:51,406 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:51,416 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=781 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/9bb28673f8be4424b378b22e60239251 2024-12-09T06:16:51,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725071417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/b93a783a192c4ece915b7bae5d652300 is 50, key is test_row_0/C:col10/1733725010161/Put/seqid=0 2024-12-09T06:16:51,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742103_1279 (size=12301) 2024-12-09T06:16:51,465 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=781 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/b93a783a192c4ece915b7bae5d652300 2024-12-09T06:16:51,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/c1737a1779b94e119970465e316e281a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c1737a1779b94e119970465e316e281a 2024-12-09T06:16:51,477 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c1737a1779b94e119970465e316e281a, entries=150, sequenceid=781, filesize=12.0 K 2024-12-09T06:16:51,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/9bb28673f8be4424b378b22e60239251 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/9bb28673f8be4424b378b22e60239251 2024-12-09T06:16:51,483 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/9bb28673f8be4424b378b22e60239251, entries=150, sequenceid=781, filesize=12.0 K 2024-12-09T06:16:51,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/b93a783a192c4ece915b7bae5d652300 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b93a783a192c4ece915b7bae5d652300 2024-12-09T06:16:51,497 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b93a783a192c4ece915b7bae5d652300, entries=150, sequenceid=781, filesize=12.0 K 2024-12-09T06:16:51,499 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=46.96 KB/48090 for f4594c92fbabea614394c10ba9359df8 in 502ms, sequenceid=781, compaction requested=false 2024-12-09T06:16:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:51,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-09T06:16:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-09T06:16:51,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-09T06:16:51,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-12-09T06:16:51,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4280 sec 2024-12-09T06:16:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:51,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:16:51,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:51,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:51,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/657089bbacc9412d87f89208d5e63ea2 is 50, key is test_row_0/A:col10/1733725011296/Put/seqid=0 2024-12-09T06:16:51,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742104_1280 (size=14741) 2024-12-09T06:16:51,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=794 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/657089bbacc9412d87f89208d5e63ea2 2024-12-09T06:16:51,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/8cf2d97302b44de5afd946602d364b29 is 50, key is test_row_0/B:col10/1733725011296/Put/seqid=0 2024-12-09T06:16:51,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742105_1281 (size=12301) 2024-12-09T06:16:51,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=794 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/8cf2d97302b44de5afd946602d364b29 2024-12-09T06:16:51,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/917b99381d6a4ef0a23d493b53cd7bc1 is 50, key is test_row_0/C:col10/1733725011296/Put/seqid=0 2024-12-09T06:16:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742106_1282 (size=12301) 2024-12-09T06:16:51,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725071703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725071702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725011695, max timestamp=1733725011707Num uniques 859; 2024-12-09T06:16:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725011695, max timestamp=1733725011707Num uniques 859; 2024-12-09T06:16:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725011695, max timestamp=1733725011707Num uniques 859; 2024-12-09T06:16:51,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725071786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725071817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725071817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725071896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725072026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725072026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=794 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/917b99381d6a4ef0a23d493b53cd7bc1 2024-12-09T06:16:52,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/657089bbacc9412d87f89208d5e63ea2 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/657089bbacc9412d87f89208d5e63ea2 2024-12-09T06:16:52,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/657089bbacc9412d87f89208d5e63ea2, entries=200, sequenceid=794, filesize=14.4 K 2024-12-09T06:16:52,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/8cf2d97302b44de5afd946602d364b29 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/8cf2d97302b44de5afd946602d364b29 2024-12-09T06:16:52,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/8cf2d97302b44de5afd946602d364b29, entries=150, sequenceid=794, filesize=12.0 K 2024-12-09T06:16:52,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/917b99381d6a4ef0a23d493b53cd7bc1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/917b99381d6a4ef0a23d493b53cd7bc1 2024-12-09T06:16:52,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/917b99381d6a4ef0a23d493b53cd7bc1, entries=150, sequenceid=794, filesize=12.0 K 2024-12-09T06:16:52,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f4594c92fbabea614394c10ba9359df8 in 474ms, sequenceid=794, compaction requested=true 2024-12-09T06:16:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:52,102 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:52,102 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:52,103 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:52,103 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:52,103 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:52,103 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:52,103 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:52,103 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:52,103 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7179a23fb5bf42fbb43249c992ad2a3a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/9bb28673f8be4424b378b22e60239251, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/8cf2d97302b44de5afd946602d364b29] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=36.7 K 2024-12-09T06:16:52,103 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e7ce5177d7f5480eb15ac1adebc0e71d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c1737a1779b94e119970465e316e281a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/657089bbacc9412d87f89208d5e63ea2] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=39.2 K 2024-12-09T06:16:52,104 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7179a23fb5bf42fbb43249c992ad2a3a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733725008910 2024-12-09T06:16:52,104 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ce5177d7f5480eb15ac1adebc0e71d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733725008910 2024-12-09T06:16:52,104 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb28673f8be4424b378b22e60239251, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=781, earliestPutTs=1733725010161 2024-12-09T06:16:52,104 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1737a1779b94e119970465e316e281a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=781, earliestPutTs=1733725010161 2024-12-09T06:16:52,104 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cf2d97302b44de5afd946602d364b29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=794, earliestPutTs=1733725011292 2024-12-09T06:16:52,104 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 657089bbacc9412d87f89208d5e63ea2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=794, earliestPutTs=1733725011292 2024-12-09T06:16:52,110 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:52,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T06:16:52,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:52,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:52,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:52,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:52,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:52,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:52,111 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/7f6a93eb9f504ba3af37310a8a6c2a4e is 50, key is test_row_0/B:col10/1733725011296/Put/seqid=0 2024-12-09T06:16:52,112 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:52,112 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/b57b998cee5047ba8ef49e41011b80e4 is 50, key is test_row_0/A:col10/1733725011296/Put/seqid=0 2024-12-09T06:16:52,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/912aaad2d2064684aafe3031992d6909 is 50, key is test_row_0/A:col10/1733725011779/Put/seqid=0 2024-12-09T06:16:52,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742107_1283 (size=13119) 2024-12-09T06:16:52,131 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/7f6a93eb9f504ba3af37310a8a6c2a4e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7f6a93eb9f504ba3af37310a8a6c2a4e 2024-12-09T06:16:52,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742108_1284 (size=13153) 2024-12-09T06:16:52,137 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into 7f6a93eb9f504ba3af37310a8a6c2a4e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:52,137 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:52,137 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=13, startTime=1733725012102; duration=0sec 2024-12-09T06:16:52,137 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:52,137 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:52,137 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:52,138 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:52,139 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:52,139 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:52,139 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/92a51bef401f4cc8ba8365b5b94edea1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b93a783a192c4ece915b7bae5d652300, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/917b99381d6a4ef0a23d493b53cd7bc1] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=36.8 K 2024-12-09T06:16:52,139 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 92a51bef401f4cc8ba8365b5b94edea1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1733725008910 2024-12-09T06:16:52,141 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b93a783a192c4ece915b7bae5d652300, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=781, earliestPutTs=1733725010161 2024-12-09T06:16:52,142 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 917b99381d6a4ef0a23d493b53cd7bc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=794, earliestPutTs=1733725011292 2024-12-09T06:16:52,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742109_1285 (size=14741) 2024-12-09T06:16:52,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/912aaad2d2064684aafe3031992d6909 2024-12-09T06:16:52,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725072146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f6cd01435b804729ba455ffab62428b9 is 50, key is test_row_0/B:col10/1733725011779/Put/seqid=0 2024-12-09T06:16:52,154 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:52,154 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/bdf294b82f2f4c29a95f82f8ce62fed2 is 50, key is test_row_0/C:col10/1733725011296/Put/seqid=0 2024-12-09T06:16:52,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742110_1286 (size=12301) 2024-12-09T06:16:52,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742111_1287 (size=13153) 2024-12-09T06:16:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T06:16:52,227 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:52,227 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-09T06:16:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T06:16:52,229 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:52,230 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:52,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725072257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725072316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725072317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,320 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in f4594c92fbabea614394c10ba9359df8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,320 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in f4594c92fbabea614394c10ba9359df8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T06:16:52,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725072337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725072337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T06:16:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:52,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:52,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725072466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T06:16:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:52,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T06:16:52,550 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/b57b998cee5047ba8ef49e41011b80e4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/b57b998cee5047ba8ef49e41011b80e4 2024-12-09T06:16:52,557 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into b57b998cee5047ba8ef49e41011b80e4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:52,557 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:52,557 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=13, startTime=1733725012102; duration=0sec 2024-12-09T06:16:52,557 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:52,557 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:52,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f6cd01435b804729ba455ffab62428b9 2024-12-09T06:16:52,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/b7fedd381c2847e48092ee2f26293c36 is 50, key is test_row_0/C:col10/1733725011779/Put/seqid=0 2024-12-09T06:16:52,573 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/bdf294b82f2f4c29a95f82f8ce62fed2 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/bdf294b82f2f4c29a95f82f8ce62fed2 2024-12-09T06:16:52,584 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into bdf294b82f2f4c29a95f82f8ce62fed2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:52,584 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:52,584 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=13, startTime=1733725012102; duration=0sec 2024-12-09T06:16:52,584 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:52,584 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:52,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742112_1288 (size=12301) 2024-12-09T06:16:52,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/b7fedd381c2847e48092ee2f26293c36 2024-12-09T06:16:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/912aaad2d2064684aafe3031992d6909 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/912aaad2d2064684aafe3031992d6909 2024-12-09T06:16:52,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/912aaad2d2064684aafe3031992d6909, entries=200, sequenceid=831, filesize=14.4 K 2024-12-09T06:16:52,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f6cd01435b804729ba455ffab62428b9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f6cd01435b804729ba455ffab62428b9 2024-12-09T06:16:52,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f6cd01435b804729ba455ffab62428b9, entries=150, sequenceid=831, filesize=12.0 K 2024-12-09T06:16:52,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/b7fedd381c2847e48092ee2f26293c36 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b7fedd381c2847e48092ee2f26293c36 2024-12-09T06:16:52,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b7fedd381c2847e48092ee2f26293c36, entries=150, sequenceid=831, filesize=12.0 K 2024-12-09T06:16:52,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f4594c92fbabea614394c10ba9359df8 in 514ms, sequenceid=831, compaction requested=false 2024-12-09T06:16:52,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:52,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T06:16:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:52,690 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1f797bd64a2c458783a9e6b231494b24 is 50, key is test_row_0/A:col10/1733725012127/Put/seqid=0 2024-12-09T06:16:52,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742113_1289 (size=9857) 2024-12-09T06:16:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:52,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:52,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725012814, max timestamp=1733725012847Num uniques 859; 2024-12-09T06:16:52,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725012814, max timestamp=1733725012847Num uniques 859; 2024-12-09T06:16:52,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:52,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:52,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725012814, max timestamp=1733725012847Num uniques 859; 2024-12-09T06:16:52,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:16:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T06:16:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725072898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725072903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725072903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725073006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725073017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725073017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,112 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1f797bd64a2c458783a9e6b231494b24 2024-12-09T06:16:53,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/5437c6a2b17d41169760a573d9ddae3d is 50, key is test_row_0/B:col10/1733725012127/Put/seqid=0 2024-12-09T06:16:53,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742114_1290 (size=9857) 2024-12-09T06:16:53,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725073216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725073226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725073226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T06:16:53,524 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/5437c6a2b17d41169760a573d9ddae3d 2024-12-09T06:16:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725073526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725073537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725073537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:53,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/8243e3b4d7534d5391611b0ddd7964c8 is 50, key is test_row_0/C:col10/1733725012127/Put/seqid=0 2024-12-09T06:16:53,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742115_1291 (size=9857) 2024-12-09T06:16:53,965 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/8243e3b4d7534d5391611b0ddd7964c8 2024-12-09T06:16:53,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1f797bd64a2c458783a9e6b231494b24 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1f797bd64a2c458783a9e6b231494b24 2024-12-09T06:16:53,986 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1f797bd64a2c458783a9e6b231494b24, entries=100, sequenceid=844, filesize=9.6 K 2024-12-09T06:16:53,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/5437c6a2b17d41169760a573d9ddae3d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/5437c6a2b17d41169760a573d9ddae3d 2024-12-09T06:16:54,014 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/5437c6a2b17d41169760a573d9ddae3d, entries=100, sequenceid=844, filesize=9.6 K 2024-12-09T06:16:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/8243e3b4d7534d5391611b0ddd7964c8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/8243e3b4d7534d5391611b0ddd7964c8 2024-12-09T06:16:54,022 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/8243e3b4d7534d5391611b0ddd7964c8, entries=100, sequenceid=844, filesize=9.6 K 2024-12-09T06:16:54,026 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f4594c92fbabea614394c10ba9359df8 in 1337ms, sequenceid=844, compaction requested=true 2024-12-09T06:16:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-09T06:16:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-09T06:16:54,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-09T06:16:54,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-12-09T06:16:54,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.8080 sec 2024-12-09T06:16:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:54,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-09T06:16:54,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:54,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:16:54,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:54,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:16:54,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:54,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:16:54,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4c8bdc68b07b45608c1895c79af12f1d is 50, key is test_row_0/A:col10/1733725012901/Put/seqid=0 2024-12-09T06:16:54,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742116_1292 (size=14741) 2024-12-09T06:16:54,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725074069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4c8bdc68b07b45608c1895c79af12f1d 2024-12-09T06:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725074086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725074087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/b023fe9f52bb45648c81e078b5e07960 is 50, key is test_row_0/B:col10/1733725012901/Put/seqid=0 2024-12-09T06:16:54,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742117_1293 (size=12301) 2024-12-09T06:16:54,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725074187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725074203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725074207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60420 deadline: 1733725074336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60432 deadline: 1733725074336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,341 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in f4594c92fbabea614394c10ba9359df8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,342 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in f4594c92fbabea614394c10ba9359df8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:16:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T06:16:54,377 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:54,381 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-09T06:16:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T06:16:54,387 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:54,389 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:54,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:54,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725074409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725074416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725074418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T06:16:54,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T06:16:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:54,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:54,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:54,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/b023fe9f52bb45648c81e078b5e07960 2024-12-09T06:16:54,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/361266d2de3d41fa9f01d39a4b42f319 is 50, key is test_row_0/C:col10/1733725012901/Put/seqid=0 2024-12-09T06:16:54,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742118_1294 (size=12301) 2024-12-09T06:16:54,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/361266d2de3d41fa9f01d39a4b42f319 2024-12-09T06:16:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T06:16:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4c8bdc68b07b45608c1895c79af12f1d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4c8bdc68b07b45608c1895c79af12f1d 2024-12-09T06:16:54,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T06:16:54,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:54,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:54,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:54,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:54,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725074720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725074726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725074726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:54,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4c8bdc68b07b45608c1895c79af12f1d, entries=200, sequenceid=882, filesize=14.4 K 2024-12-09T06:16:54,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/b023fe9f52bb45648c81e078b5e07960 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/b023fe9f52bb45648c81e078b5e07960 2024-12-09T06:16:54,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/b023fe9f52bb45648c81e078b5e07960, entries=150, sequenceid=882, filesize=12.0 K 2024-12-09T06:16:54,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/361266d2de3d41fa9f01d39a4b42f319 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/361266d2de3d41fa9f01d39a4b42f319 2024-12-09T06:16:54,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/361266d2de3d41fa9f01d39a4b42f319, entries=150, sequenceid=882, filesize=12.0 K 2024-12-09T06:16:54,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=46.96 KB/48090 for f4594c92fbabea614394c10ba9359df8 in 756ms, sequenceid=882, compaction requested=true 2024-12-09T06:16:54,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:54,801 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:16:54,801 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:16:54,805 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52492 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:54,805 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:54,805 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:54,805 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/b57b998cee5047ba8ef49e41011b80e4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/912aaad2d2064684aafe3031992d6909, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1f797bd64a2c458783a9e6b231494b24, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4c8bdc68b07b45608c1895c79af12f1d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=51.3 K 2024-12-09T06:16:54,805 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:16:54,805 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:54,806 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:54,806 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7f6a93eb9f504ba3af37310a8a6c2a4e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f6cd01435b804729ba455ffab62428b9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/5437c6a2b17d41169760a573d9ddae3d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/b023fe9f52bb45648c81e078b5e07960] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=46.5 K 2024-12-09T06:16:54,806 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57b998cee5047ba8ef49e41011b80e4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=794, earliestPutTs=1733725011292 2024-12-09T06:16:54,807 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f6a93eb9f504ba3af37310a8a6c2a4e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=794, earliestPutTs=1733725011292 2024-12-09T06:16:54,807 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 912aaad2d2064684aafe3031992d6909, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1733725011775 2024-12-09T06:16:54,807 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f6cd01435b804729ba455ffab62428b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1733725011775 2024-12-09T06:16:54,808 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f797bd64a2c458783a9e6b231494b24, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1733725012127 2024-12-09T06:16:54,809 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5437c6a2b17d41169760a573d9ddae3d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1733725012127 2024-12-09T06:16:54,809 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c8bdc68b07b45608c1895c79af12f1d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1733725012895 2024-12-09T06:16:54,810 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b023fe9f52bb45648c81e078b5e07960, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1733725012900 2024-12-09T06:16:54,827 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:54,828 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/a81ea8db9ca44a8fa8ae40a79391872d is 50, key is test_row_0/A:col10/1733725012901/Put/seqid=0 2024-12-09T06:16:54,839 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:54,840 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f0d8d4d38a384ecab98fa8ac6677d476 is 50, key is test_row_0/B:col10/1733725012901/Put/seqid=0 2024-12-09T06:16:54,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742120_1296 (size=13255) 2024-12-09T06:16:54,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742119_1295 (size=13289) 2024-12-09T06:16:54,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T06:16:54,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:54,879 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:54,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4865485da1c349328071421dc4fabce6 is 50, key is test_row_0/A:col10/1733725014079/Put/seqid=0 2024-12-09T06:16:54,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742121_1297 (size=7415) 2024-12-09T06:16:54,915 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=892 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4865485da1c349328071421dc4fabce6 2024-12-09T06:16:54,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b is 50, key is test_row_0/B:col10/1733725014079/Put/seqid=0 2024-12-09T06:16:54,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742122_1298 (size=7415) 2024-12-09T06:16:54,987 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=892 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b 2024-12-09T06:16:55,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/2bedc60da1ee457b8c7cb75ae352cc99 is 50, key is test_row_0/C:col10/1733725014079/Put/seqid=0 2024-12-09T06:16:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T06:16:55,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742123_1299 (size=7415) 2024-12-09T06:16:55,065 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=892 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/2bedc60da1ee457b8c7cb75ae352cc99 2024-12-09T06:16:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/4865485da1c349328071421dc4fabce6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4865485da1c349328071421dc4fabce6 2024-12-09T06:16:55,084 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4865485da1c349328071421dc4fabce6, entries=50, sequenceid=892, filesize=7.2 K 2024-12-09T06:16:55,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b 2024-12-09T06:16:55,095 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b, entries=50, sequenceid=892, filesize=7.2 K 2024-12-09T06:16:55,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/2bedc60da1ee457b8c7cb75ae352cc99 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2bedc60da1ee457b8c7cb75ae352cc99 2024-12-09T06:16:55,105 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2bedc60da1ee457b8c7cb75ae352cc99, entries=50, sequenceid=892, filesize=7.2 K 2024-12-09T06:16:55,106 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for f4594c92fbabea614394c10ba9359df8 in 227ms, sequenceid=892, compaction requested=true 2024-12-09T06:16:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-09T06:16:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-09T06:16:55,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-09T06:16:55,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 725 msec 2024-12-09T06:16:55,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 742 msec 2024-12-09T06:16:55,272 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/f0d8d4d38a384ecab98fa8ac6677d476 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f0d8d4d38a384ecab98fa8ac6677d476 2024-12-09T06:16:55,279 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/a81ea8db9ca44a8fa8ae40a79391872d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/a81ea8db9ca44a8fa8ae40a79391872d 2024-12-09T06:16:55,281 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into f0d8d4d38a384ecab98fa8ac6677d476(size=12.9 K), total size for store is 20.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:55,281 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:55,281 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=12, startTime=1733725014801; duration=0sec 2024-12-09T06:16:55,281 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:55,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:16:55,281 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:55,281 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T06:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:55,284 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 55027 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T06:16:55,284 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/C is initiating minor compaction (all files) 2024-12-09T06:16:55,284 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/C in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:55,284 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/bdf294b82f2f4c29a95f82f8ce62fed2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b7fedd381c2847e48092ee2f26293c36, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/8243e3b4d7534d5391611b0ddd7964c8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/361266d2de3d41fa9f01d39a4b42f319, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2bedc60da1ee457b8c7cb75ae352cc99] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=53.7 K 2024-12-09T06:16:55,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/3f6bfdf893314e8c8d3016fdb7f996fd is 50, key is test_row_0/A:col10/1733725015250/Put/seqid=0 2024-12-09T06:16:55,291 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into a81ea8db9ca44a8fa8ae40a79391872d(size=13.0 K), total size for store is 20.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:55,291 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:55,291 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=12, startTime=1733725014801; duration=0sec 2024-12-09T06:16:55,291 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:55,291 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:55,293 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf294b82f2f4c29a95f82f8ce62fed2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=794, earliestPutTs=1733725011292 2024-12-09T06:16:55,297 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b7fedd381c2847e48092ee2f26293c36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1733725011775 2024-12-09T06:16:55,297 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8243e3b4d7534d5391611b0ddd7964c8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=844, earliestPutTs=1733725012127 2024-12-09T06:16:55,298 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 361266d2de3d41fa9f01d39a4b42f319, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1733725012900 2024-12-09T06:16:55,299 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bedc60da1ee457b8c7cb75ae352cc99, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=892, earliestPutTs=1733725014079 2024-12-09T06:16:55,338 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#C#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:55,338 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/c425034e74a2492890e4f7203971bc06 is 50, key is test_row_0/C:col10/1733725014079/Put/seqid=0 2024-12-09T06:16:55,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725015340, max timestamp=1733725015361Num uniques 859; 2024-12-09T06:16:55,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:16:55,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742124_1300 (size=14741) 2024-12-09T06:16:55,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742125_1301 (size=13323) 2024-12-09T06:16:55,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725075383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725075389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725075389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725075506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725075506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725075506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T06:16:55,537 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:55,538 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:16:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-09T06:16:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T06:16:55,545 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:16:55,549 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:16:55,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:16:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T06:16:55,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T06:16:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:55,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:55,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725075718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725075718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725075721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:55,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=903 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/3f6bfdf893314e8c8d3016fdb7f996fd 2024-12-09T06:16:55,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/a65cce7b966246468f76d60bc46ef76c is 50, key is test_row_0/B:col10/1733725015250/Put/seqid=0 2024-12-09T06:16:55,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742126_1302 (size=12301) 2024-12-09T06:16:55,822 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/c425034e74a2492890e4f7203971bc06 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/c425034e74a2492890e4f7203971bc06 2024-12-09T06:16:55,852 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f4594c92fbabea614394c10ba9359df8/C of f4594c92fbabea614394c10ba9359df8 into c425034e74a2492890e4f7203971bc06(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:55,852 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:55,852 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/C, priority=11, startTime=1733725014801; duration=0sec 2024-12-09T06:16:55,852 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:55,852 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T06:16:55,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T06:16:55,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:55,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:55,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:55,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:55,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:56,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T06:16:56,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:56,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:56,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:56,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:56,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:56,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60424 deadline: 1733725076027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:56,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60362 deadline: 1733725076027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:56,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:16:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60456 deadline: 1733725076029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:16:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:16:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T06:16:56,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T06:16:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. as already flushing 2024-12-09T06:16:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:56,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:16:56,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=903 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/a65cce7b966246468f76d60bc46ef76c 2024-12-09T06:16:56,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/13e0f9f4828f4b8a971942e9075f6006 is 50, key is test_row_0/C:col10/1733725015250/Put/seqid=0 2024-12-09T06:16:56,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742127_1303 (size=12301) 2024-12-09T06:16:56,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=903 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/13e0f9f4828f4b8a971942e9075f6006 2024-12-09T06:16:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/3f6bfdf893314e8c8d3016fdb7f996fd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3f6bfdf893314e8c8d3016fdb7f996fd 2024-12-09T06:16:56,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3f6bfdf893314e8c8d3016fdb7f996fd, entries=200, sequenceid=903, filesize=14.4 K 2024-12-09T06:16:56,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/a65cce7b966246468f76d60bc46ef76c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a65cce7b966246468f76d60bc46ef76c 2024-12-09T06:16:56,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a65cce7b966246468f76d60bc46ef76c, entries=150, sequenceid=903, filesize=12.0 K 2024-12-09T06:16:56,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/13e0f9f4828f4b8a971942e9075f6006 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/13e0f9f4828f4b8a971942e9075f6006 2024-12-09T06:16:56,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/13e0f9f4828f4b8a971942e9075f6006, entries=150, sequenceid=903, filesize=12.0 K 2024-12-09T06:16:56,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f4594c92fbabea614394c10ba9359df8 in 1016ms, sequenceid=903, compaction requested=true 2024-12-09T06:16:56,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:56,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:16:56,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:56,298 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:56,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:16:56,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:56,298 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:16:56,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4594c92fbabea614394c10ba9359df8:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:16:56,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:56,305 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:56,305 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35445 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:16:56,305 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/B is initiating minor compaction (all files) 2024-12-09T06:16:56,305 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): f4594c92fbabea614394c10ba9359df8/A is initiating minor compaction (all files) 2024-12-09T06:16:56,306 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/B in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:56,306 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f0d8d4d38a384ecab98fa8ac6677d476, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a65cce7b966246468f76d60bc46ef76c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=32.2 K 2024-12-09T06:16:56,306 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f4594c92fbabea614394c10ba9359df8/A in TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:56,306 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/a81ea8db9ca44a8fa8ae40a79391872d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4865485da1c349328071421dc4fabce6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3f6bfdf893314e8c8d3016fdb7f996fd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp, totalSize=34.6 K 2024-12-09T06:16:56,306 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f0d8d4d38a384ecab98fa8ac6677d476, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1733725012900 2024-12-09T06:16:56,307 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81ea8db9ca44a8fa8ae40a79391872d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1733725012900 2024-12-09T06:16:56,307 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4865485da1c349328071421dc4fabce6, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=892, earliestPutTs=1733725014079 2024-12-09T06:16:56,307 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f6bfdf893314e8c8d3016fdb7f996fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=903, earliestPutTs=1733725015250 2024-12-09T06:16:56,313 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd8b1bc63a440dbb8d30fdf56b8cf7b, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=892, earliestPutTs=1733725014079 2024-12-09T06:16:56,314 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a65cce7b966246468f76d60bc46ef76c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=903, earliestPutTs=1733725015250 2024-12-09T06:16:56,332 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:56,333 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/c994a5333e2a4548b2a2afe9064a3cd7 is 50, key is test_row_0/A:col10/1733725015250/Put/seqid=0 2024-12-09T06:16:56,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T06:16:56,336 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4594c92fbabea614394c10ba9359df8#B#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:16:56,336 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/2d1c9d5a244b4df98cde19ef1dc35243 is 50, key is test_row_0/B:col10/1733725015250/Put/seqid=0 2024-12-09T06:16:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:56,338 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=134.18 KB heapSize=391.80 KB 2024-12-09T06:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:16:56,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/cd6f016ed1e84c108aa2b44d190e44f1 is 50, key is test_row_0/A:col10/1733725015382/Put/seqid=0 2024-12-09T06:16:56,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742128_1304 (size=13391) 2024-12-09T06:16:56,406 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/c994a5333e2a4548b2a2afe9064a3cd7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c994a5333e2a4548b2a2afe9064a3cd7 2024-12-09T06:16:56,415 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/A of f4594c92fbabea614394c10ba9359df8 into c994a5333e2a4548b2a2afe9064a3cd7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:56,415 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:56,416 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/A, priority=13, startTime=1733725016297; duration=0sec 2024-12-09T06:16:56,416 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:16:56,416 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:A 2024-12-09T06:16:56,416 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T06:16:56,418 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T06:16:56,418 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T06:16:56,418 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. because compaction request was cancelled 2024-12-09T06:16:56,418 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:C 2024-12-09T06:16:56,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742129_1305 (size=13357) 2024-12-09T06:16:56,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742130_1306 (size=12301) 2024-12-09T06:16:56,426 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=932 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/cd6f016ed1e84c108aa2b44d190e44f1 2024-12-09T06:16:56,429 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/2d1c9d5a244b4df98cde19ef1dc35243 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2d1c9d5a244b4df98cde19ef1dc35243 2024-12-09T06:16:56,445 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f4594c92fbabea614394c10ba9359df8/B of f4594c92fbabea614394c10ba9359df8 into 2d1c9d5a244b4df98cde19ef1dc35243(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:16:56,445 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:56,445 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., storeName=f4594c92fbabea614394c10ba9359df8/B, priority=13, startTime=1733725016298; duration=0sec 2024-12-09T06:16:56,445 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:16:56,445 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4594c92fbabea614394c10ba9359df8:B 2024-12-09T06:16:56,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/dbe9876461b147d491f4da78956e3fde is 50, key is test_row_0/B:col10/1733725015382/Put/seqid=0 2024-12-09T06:16:56,476 INFO [Thread-1002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1002. 2024-12-09T06:16:56,476 DEBUG [Thread-1002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:56,476 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,476 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:56,477 INFO [Thread-1004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1004. 2024-12-09T06:16:56,477 DEBUG [Thread-1004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:56,477 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,478 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742131_1307 (size=12301) 2024-12-09T06:16:56,478 INFO [Thread-1000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1000. 2024-12-09T06:16:56,478 DEBUG [Thread-1000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:56,478 INFO [Thread-1008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1008. 2024-12-09T06:16:56,478 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,478 DEBUG [Thread-1008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:56,478 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,478 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,478 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:56,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:56,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:56,481 INFO [Thread-1006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1006. 2024-12-09T06:16:56,481 DEBUG [Thread-1006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:56,481 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,481 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:56,488 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=932 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/dbe9876461b147d491f4da78956e3fde 2024-12-09T06:16:56,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/884ad64dbb4b42aba0ee5b50a139245b is 50, key is test_row_0/C:col10/1733725015382/Put/seqid=0 2024-12-09T06:16:56,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742132_1308 (size=12301) 2024-12-09T06:16:56,511 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=932 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/884ad64dbb4b42aba0ee5b50a139245b 2024-12-09T06:16:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/cd6f016ed1e84c108aa2b44d190e44f1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/cd6f016ed1e84c108aa2b44d190e44f1 2024-12-09T06:16:56,520 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/cd6f016ed1e84c108aa2b44d190e44f1, entries=150, sequenceid=932, filesize=12.0 K 2024-12-09T06:16:56,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/dbe9876461b147d491f4da78956e3fde as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/dbe9876461b147d491f4da78956e3fde 2024-12-09T06:16:56,526 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/dbe9876461b147d491f4da78956e3fde, entries=150, sequenceid=932, filesize=12.0 K 2024-12-09T06:16:56,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/884ad64dbb4b42aba0ee5b50a139245b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/884ad64dbb4b42aba0ee5b50a139245b 2024-12-09T06:16:56,531 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/884ad64dbb4b42aba0ee5b50a139245b, entries=150, sequenceid=932, filesize=12.0 K 2024-12-09T06:16:56,532 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~391.75 KB/401152, currentSize=0 B/0 for f4594c92fbabea614394c10ba9359df8 in 195ms, sequenceid=932, compaction requested=true 2024-12-09T06:16:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f4594c92fbabea614394c10ba9359df8: 2024-12-09T06:16:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-09T06:16:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-09T06:16:56,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-09T06:16:56,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 984 msec 2024-12-09T06:16:56,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 997 msec 2024-12-09T06:16:56,539 INFO [Thread-991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-991. 2024-12-09T06:16:56,539 DEBUG [Thread-991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:56,539 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,540 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:56,540 INFO [Thread-995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-995. 2024-12-09T06:16:56,540 DEBUG [Thread-995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:56,540 DEBUG [Thread-995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,540 DEBUG [Thread-995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:56,540 INFO [Thread-993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-993. 2024-12-09T06:16:56,541 DEBUG [Thread-993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:56,541 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,541 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:56,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T06:16:56,677 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:16:58,220 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T06:16:58,389 INFO [Thread-989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-989. 2024-12-09T06:16:58,389 INFO [Thread-997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-997. 2024-12-09T06:16:58,389 DEBUG [Thread-989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:58,389 DEBUG [Thread-997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:16:58,389 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:58,389 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:58,389 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:58,389 DEBUG [Thread-997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:58,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:58,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 171 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 239 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 168 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2689 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2720 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2646 2024-12-09T06:16:58,389 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2645 2024-12-09T06:16:58,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2670 2024-12-09T06:16:58,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T06:16:58,390 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T06:16:58,390 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:16:58,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:58,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:16:58,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:16:58,391 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T06:16:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T06:16:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T06:16:58,394 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725018394"}]},"ts":"1733725018394"} 2024-12-09T06:16:58,395 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T06:16:58,395 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T06:16:58,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:16:58,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, UNASSIGN}] 2024-12-09T06:16:58,398 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, UNASSIGN 2024-12-09T06:16:58,398 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=f4594c92fbabea614394c10ba9359df8, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:16:58,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, UNASSIGN because future has completed 2024-12-09T06:16:58,400 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:16:58,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:16:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T06:16:58,552 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing f4594c92fbabea614394c10ba9359df8, disabling compactions & flushes 2024-12-09T06:16:58,552 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. after waiting 0 ms 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:58,552 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing f4594c92fbabea614394c10ba9359df8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=A 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=B 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f4594c92fbabea614394c10ba9359df8, store=C 2024-12-09T06:16:58,552 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:16:58,556 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1cb27e7802dd436da1b83b5a10dcbd55 is 50, key is test_row_0/A:col10/1733725018387/Put/seqid=0 2024-12-09T06:16:58,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742133_1309 (size=9857) 2024-12-09T06:16:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T06:16:58,960 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1cb27e7802dd436da1b83b5a10dcbd55 2024-12-09T06:16:58,972 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/10cb893824cd4bcc8c3c11a03956cb1d is 50, key is test_row_0/B:col10/1733725018387/Put/seqid=0 2024-12-09T06:16:58,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742134_1310 (size=9857) 2024-12-09T06:16:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T06:16:59,376 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/10cb893824cd4bcc8c3c11a03956cb1d 2024-12-09T06:16:59,382 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/35bedef8796340c289d19cc5221a1e01 is 50, key is test_row_0/C:col10/1733725018387/Put/seqid=0 2024-12-09T06:16:59,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742135_1311 (size=9857) 2024-12-09T06:16:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T06:16:59,786 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/35bedef8796340c289d19cc5221a1e01 2024-12-09T06:16:59,790 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/A/1cb27e7802dd436da1b83b5a10dcbd55 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cb27e7802dd436da1b83b5a10dcbd55 2024-12-09T06:16:59,794 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cb27e7802dd436da1b83b5a10dcbd55, entries=100, sequenceid=942, filesize=9.6 K 2024-12-09T06:16:59,795 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/B/10cb893824cd4bcc8c3c11a03956cb1d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/10cb893824cd4bcc8c3c11a03956cb1d 2024-12-09T06:16:59,799 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/10cb893824cd4bcc8c3c11a03956cb1d, entries=100, sequenceid=942, filesize=9.6 K 2024-12-09T06:16:59,800 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/.tmp/C/35bedef8796340c289d19cc5221a1e01 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/35bedef8796340c289d19cc5221a1e01 2024-12-09T06:16:59,804 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/35bedef8796340c289d19cc5221a1e01, entries=100, sequenceid=942, filesize=9.6 K 2024-12-09T06:16:59,805 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f4594c92fbabea614394c10ba9359df8 in 1253ms, sequenceid=942, compaction requested=true 2024-12-09T06:16:59,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1e7bf4c100054cbdb063ea24e8ff4458, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cfa94e2e8ac48cb854a5badc799c54b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/43aa22bf13134903b1e377f50a9f0318, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3a038e2f29314551810d0a0c34b0107e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/0aaf00549b6d4e67bedc8d90befedadd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fa633d9dfeae48238fdeb47243bb61e1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/5465170d0d014985b09eba45552d7bfa, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/24a6328b9a8749a98cc557adaf0b9410, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/7ce77e76bf2745fab1728dc7bec1ca38, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1ea915f89f5843ae9468d3295740aa51, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/217bc51a9a114cf5be4d411074a732ab, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/550979bb384e428c9c831663c32f2164, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c5a90b54a15d4e24b0cdf9e86f6b0026, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e1673c33d38a4616bb9116d2731bef94, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4ffd3c8b0ce04ac88f126fc8e28f79ab, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/69c306cb6cd84507b84fbbf0cbe1191b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/f117b7d3a3954791b547f7dbbc962612, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fee2ca0afd644ffea63ea5a06f1bfff4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/2789f6127a8644c29c1f8c25e6b60b3f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d877a8afc9fe49abae2e0e0f613133cf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d1e21b3ebb204c7ba6f3b7346044aab6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e7ce5177d7f5480eb15ac1adebc0e71d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/30cc4d7fa9f94b4f90783c86333c4ba0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c1737a1779b94e119970465e316e281a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/657089bbacc9412d87f89208d5e63ea2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/b57b998cee5047ba8ef49e41011b80e4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/912aaad2d2064684aafe3031992d6909, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1f797bd64a2c458783a9e6b231494b24, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4c8bdc68b07b45608c1895c79af12f1d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/a81ea8db9ca44a8fa8ae40a79391872d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4865485da1c349328071421dc4fabce6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3f6bfdf893314e8c8d3016fdb7f996fd] to archive 2024-12-09T06:16:59,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:59,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1e7bf4c100054cbdb063ea24e8ff4458 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1e7bf4c100054cbdb063ea24e8ff4458 2024-12-09T06:16:59,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cfa94e2e8ac48cb854a5badc799c54b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cfa94e2e8ac48cb854a5badc799c54b 2024-12-09T06:16:59,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/43aa22bf13134903b1e377f50a9f0318 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/43aa22bf13134903b1e377f50a9f0318 2024-12-09T06:16:59,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3a038e2f29314551810d0a0c34b0107e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3a038e2f29314551810d0a0c34b0107e 2024-12-09T06:16:59,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/0aaf00549b6d4e67bedc8d90befedadd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/0aaf00549b6d4e67bedc8d90befedadd 2024-12-09T06:16:59,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fa633d9dfeae48238fdeb47243bb61e1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fa633d9dfeae48238fdeb47243bb61e1 2024-12-09T06:16:59,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/5465170d0d014985b09eba45552d7bfa to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/5465170d0d014985b09eba45552d7bfa 2024-12-09T06:16:59,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/24a6328b9a8749a98cc557adaf0b9410 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/24a6328b9a8749a98cc557adaf0b9410 2024-12-09T06:16:59,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/7ce77e76bf2745fab1728dc7bec1ca38 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/7ce77e76bf2745fab1728dc7bec1ca38 2024-12-09T06:16:59,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1ea915f89f5843ae9468d3295740aa51 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1ea915f89f5843ae9468d3295740aa51 2024-12-09T06:16:59,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/217bc51a9a114cf5be4d411074a732ab to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/217bc51a9a114cf5be4d411074a732ab 2024-12-09T06:16:59,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/550979bb384e428c9c831663c32f2164 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/550979bb384e428c9c831663c32f2164 2024-12-09T06:16:59,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c5a90b54a15d4e24b0cdf9e86f6b0026 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c5a90b54a15d4e24b0cdf9e86f6b0026 2024-12-09T06:16:59,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e1673c33d38a4616bb9116d2731bef94 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e1673c33d38a4616bb9116d2731bef94 2024-12-09T06:16:59,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4ffd3c8b0ce04ac88f126fc8e28f79ab to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4ffd3c8b0ce04ac88f126fc8e28f79ab 2024-12-09T06:16:59,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/69c306cb6cd84507b84fbbf0cbe1191b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/69c306cb6cd84507b84fbbf0cbe1191b 2024-12-09T06:16:59,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/f117b7d3a3954791b547f7dbbc962612 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/f117b7d3a3954791b547f7dbbc962612 2024-12-09T06:16:59,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fee2ca0afd644ffea63ea5a06f1bfff4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/fee2ca0afd644ffea63ea5a06f1bfff4 2024-12-09T06:16:59,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/2789f6127a8644c29c1f8c25e6b60b3f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/2789f6127a8644c29c1f8c25e6b60b3f 2024-12-09T06:16:59,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d877a8afc9fe49abae2e0e0f613133cf to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d877a8afc9fe49abae2e0e0f613133cf 2024-12-09T06:16:59,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d1e21b3ebb204c7ba6f3b7346044aab6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/d1e21b3ebb204c7ba6f3b7346044aab6 2024-12-09T06:16:59,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e7ce5177d7f5480eb15ac1adebc0e71d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/e7ce5177d7f5480eb15ac1adebc0e71d 2024-12-09T06:16:59,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/30cc4d7fa9f94b4f90783c86333c4ba0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/30cc4d7fa9f94b4f90783c86333c4ba0 2024-12-09T06:16:59,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c1737a1779b94e119970465e316e281a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c1737a1779b94e119970465e316e281a 2024-12-09T06:16:59,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/657089bbacc9412d87f89208d5e63ea2 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/657089bbacc9412d87f89208d5e63ea2 2024-12-09T06:16:59,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/b57b998cee5047ba8ef49e41011b80e4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/b57b998cee5047ba8ef49e41011b80e4 2024-12-09T06:16:59,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/912aaad2d2064684aafe3031992d6909 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/912aaad2d2064684aafe3031992d6909 2024-12-09T06:16:59,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1f797bd64a2c458783a9e6b231494b24 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1f797bd64a2c458783a9e6b231494b24 2024-12-09T06:16:59,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4c8bdc68b07b45608c1895c79af12f1d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4c8bdc68b07b45608c1895c79af12f1d 2024-12-09T06:16:59,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/a81ea8db9ca44a8fa8ae40a79391872d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/a81ea8db9ca44a8fa8ae40a79391872d 2024-12-09T06:16:59,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4865485da1c349328071421dc4fabce6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/4865485da1c349328071421dc4fabce6 2024-12-09T06:16:59,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3f6bfdf893314e8c8d3016fdb7f996fd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/3f6bfdf893314e8c8d3016fdb7f996fd 2024-12-09T06:16:59,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/50b1cf764b6744f29efd362bc613575a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/35b33e6b75924fbe808a78c96b26f95f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0d0627c9cb4b41cc92016f3c7b75977d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d59e98b38a06441893997d036cda005c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/92fefa3305584dc4932d0a8575d8ed92, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0af493373c1742aba685277c195d46c5, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2ef9e49ae1a464ca2a72a5ea305644f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2c0733b7c3442218a4aa84e0e41c4c3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/bd2e692c06e44466967689761ad8bf6e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2f1a9ba73ff1465face023dac361cae0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f1fc9cc9fa2548b6a8277060aad8ce4f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/fa7670b1b2bc44e6b006d45d38d69efe, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2cd35bb49b1041ad962ec4bcbec50a25, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/c45fb704708241d9a89fa81fe4e73150, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d26559939e2c42d2ae2121c799991a3d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/51fd74161d2a471c984fe25c3b6028f7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/6b1c97801562424281b40d1f14bbe905, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/70eaa425a31c47acb6ec414f95d72dec, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/112d93884aab49ab84e16553aee521fc, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f9affa624cb0411195b04c294aa95750, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7179a23fb5bf42fbb43249c992ad2a3a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ca19e0094e9f46bda16b28c09a6e4948, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/9bb28673f8be4424b378b22e60239251, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7f6a93eb9f504ba3af37310a8a6c2a4e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/8cf2d97302b44de5afd946602d364b29, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f6cd01435b804729ba455ffab62428b9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/5437c6a2b17d41169760a573d9ddae3d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f0d8d4d38a384ecab98fa8ac6677d476, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/b023fe9f52bb45648c81e078b5e07960, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a65cce7b966246468f76d60bc46ef76c] to archive 2024-12-09T06:16:59,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:59,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/50b1cf764b6744f29efd362bc613575a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/50b1cf764b6744f29efd362bc613575a 2024-12-09T06:16:59,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/35b33e6b75924fbe808a78c96b26f95f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/35b33e6b75924fbe808a78c96b26f95f 2024-12-09T06:16:59,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0d0627c9cb4b41cc92016f3c7b75977d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0d0627c9cb4b41cc92016f3c7b75977d 2024-12-09T06:16:59,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d59e98b38a06441893997d036cda005c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d59e98b38a06441893997d036cda005c 2024-12-09T06:16:59,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/92fefa3305584dc4932d0a8575d8ed92 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/92fefa3305584dc4932d0a8575d8ed92 2024-12-09T06:16:59,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0af493373c1742aba685277c195d46c5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/0af493373c1742aba685277c195d46c5 2024-12-09T06:16:59,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2ef9e49ae1a464ca2a72a5ea305644f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2ef9e49ae1a464ca2a72a5ea305644f 2024-12-09T06:16:59,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2c0733b7c3442218a4aa84e0e41c4c3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a2c0733b7c3442218a4aa84e0e41c4c3 2024-12-09T06:16:59,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/bd2e692c06e44466967689761ad8bf6e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/bd2e692c06e44466967689761ad8bf6e 2024-12-09T06:16:59,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2f1a9ba73ff1465face023dac361cae0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2f1a9ba73ff1465face023dac361cae0 2024-12-09T06:16:59,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f1fc9cc9fa2548b6a8277060aad8ce4f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f1fc9cc9fa2548b6a8277060aad8ce4f 2024-12-09T06:16:59,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/fa7670b1b2bc44e6b006d45d38d69efe to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/fa7670b1b2bc44e6b006d45d38d69efe 2024-12-09T06:16:59,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2cd35bb49b1041ad962ec4bcbec50a25 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2cd35bb49b1041ad962ec4bcbec50a25 2024-12-09T06:16:59,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/c45fb704708241d9a89fa81fe4e73150 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/c45fb704708241d9a89fa81fe4e73150 2024-12-09T06:16:59,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d26559939e2c42d2ae2121c799991a3d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/d26559939e2c42d2ae2121c799991a3d 2024-12-09T06:16:59,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/51fd74161d2a471c984fe25c3b6028f7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/51fd74161d2a471c984fe25c3b6028f7 2024-12-09T06:16:59,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/6b1c97801562424281b40d1f14bbe905 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/6b1c97801562424281b40d1f14bbe905 2024-12-09T06:16:59,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/70eaa425a31c47acb6ec414f95d72dec to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/70eaa425a31c47acb6ec414f95d72dec 2024-12-09T06:16:59,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/112d93884aab49ab84e16553aee521fc to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/112d93884aab49ab84e16553aee521fc 2024-12-09T06:16:59,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f9affa624cb0411195b04c294aa95750 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f9affa624cb0411195b04c294aa95750 2024-12-09T06:16:59,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7179a23fb5bf42fbb43249c992ad2a3a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7179a23fb5bf42fbb43249c992ad2a3a 2024-12-09T06:16:59,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ca19e0094e9f46bda16b28c09a6e4948 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ca19e0094e9f46bda16b28c09a6e4948 2024-12-09T06:16:59,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/9bb28673f8be4424b378b22e60239251 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/9bb28673f8be4424b378b22e60239251 2024-12-09T06:16:59,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7f6a93eb9f504ba3af37310a8a6c2a4e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/7f6a93eb9f504ba3af37310a8a6c2a4e 2024-12-09T06:16:59,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/8cf2d97302b44de5afd946602d364b29 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/8cf2d97302b44de5afd946602d364b29 2024-12-09T06:16:59,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f6cd01435b804729ba455ffab62428b9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f6cd01435b804729ba455ffab62428b9 2024-12-09T06:16:59,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/5437c6a2b17d41169760a573d9ddae3d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/5437c6a2b17d41169760a573d9ddae3d 2024-12-09T06:16:59,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f0d8d4d38a384ecab98fa8ac6677d476 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/f0d8d4d38a384ecab98fa8ac6677d476 2024-12-09T06:16:59,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/b023fe9f52bb45648c81e078b5e07960 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/b023fe9f52bb45648c81e078b5e07960 2024-12-09T06:16:59,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/ecd8b1bc63a440dbb8d30fdf56b8cf7b 2024-12-09T06:16:59,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a65cce7b966246468f76d60bc46ef76c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/a65cce7b966246468f76d60bc46ef76c 2024-12-09T06:16:59,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ceb9c4a88af74e9eaa42bcac66cf76f2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/36a8a91602984a2bb35fa3944eb29af7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/317cc2f36b8948c7acb159fd0680d789, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ec773f05edcd40b8905efe0ac7668e15, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/7b3da2442ee7471090735258991afa6b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/d3696616279f442ea4492cc99fdb84df, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3c542f0191a146a098a954f6507d65a9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6c13df52b7d0473997a34459228f5a84, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/528e7b3b327f49b9b6190bc8c15c5202, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3141fa6cc4954499a2b1365dce26c585, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/02b0193a18f34ee1930420a920518bf9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/f9c1977527814cafa3ee3f40132ad5ff, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2d73706c211b46fe8b7adfcb3c7f3781, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/1e25e54d9bea42a1a2b82466c8d1abd7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/a2a4fa75b7d443adb240ea215f655e85, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ffc26f5647d540149481cf0e8499fd18, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/aaa28e139ebc40e28ebd597bea8ea2f4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/25881fb9a86f4dafa8f55df705fa1903, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/04e065434d954f62867b95df54fc628f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3555497bca594654aa34ace9cc5321c3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/92a51bef401f4cc8ba8365b5b94edea1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/5e485af0897e415aa7af3c4d0f285bc7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b93a783a192c4ece915b7bae5d652300, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/bdf294b82f2f4c29a95f82f8ce62fed2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/917b99381d6a4ef0a23d493b53cd7bc1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b7fedd381c2847e48092ee2f26293c36, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/8243e3b4d7534d5391611b0ddd7964c8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/361266d2de3d41fa9f01d39a4b42f319, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2bedc60da1ee457b8c7cb75ae352cc99] to archive 2024-12-09T06:16:59,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:16:59,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ceb9c4a88af74e9eaa42bcac66cf76f2 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ceb9c4a88af74e9eaa42bcac66cf76f2 2024-12-09T06:16:59,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/36a8a91602984a2bb35fa3944eb29af7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/36a8a91602984a2bb35fa3944eb29af7 2024-12-09T06:16:59,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/317cc2f36b8948c7acb159fd0680d789 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/317cc2f36b8948c7acb159fd0680d789 2024-12-09T06:16:59,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ec773f05edcd40b8905efe0ac7668e15 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ec773f05edcd40b8905efe0ac7668e15 2024-12-09T06:16:59,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/7b3da2442ee7471090735258991afa6b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/7b3da2442ee7471090735258991afa6b 2024-12-09T06:16:59,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/d3696616279f442ea4492cc99fdb84df to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/d3696616279f442ea4492cc99fdb84df 2024-12-09T06:16:59,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3c542f0191a146a098a954f6507d65a9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3c542f0191a146a098a954f6507d65a9 2024-12-09T06:16:59,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6c13df52b7d0473997a34459228f5a84 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6c13df52b7d0473997a34459228f5a84 2024-12-09T06:16:59,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/528e7b3b327f49b9b6190bc8c15c5202 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/528e7b3b327f49b9b6190bc8c15c5202 2024-12-09T06:16:59,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3141fa6cc4954499a2b1365dce26c585 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3141fa6cc4954499a2b1365dce26c585 2024-12-09T06:16:59,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/02b0193a18f34ee1930420a920518bf9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/02b0193a18f34ee1930420a920518bf9 2024-12-09T06:16:59,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/f9c1977527814cafa3ee3f40132ad5ff to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/f9c1977527814cafa3ee3f40132ad5ff 2024-12-09T06:16:59,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2d73706c211b46fe8b7adfcb3c7f3781 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2d73706c211b46fe8b7adfcb3c7f3781 2024-12-09T06:16:59,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/1e25e54d9bea42a1a2b82466c8d1abd7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/1e25e54d9bea42a1a2b82466c8d1abd7 2024-12-09T06:16:59,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/6b3ab6ac5aaa4dd6a82a154b0eca6ee1 2024-12-09T06:16:59,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/a2a4fa75b7d443adb240ea215f655e85 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/a2a4fa75b7d443adb240ea215f655e85 2024-12-09T06:16:59,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ffc26f5647d540149481cf0e8499fd18 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/ffc26f5647d540149481cf0e8499fd18 2024-12-09T06:16:59,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/aaa28e139ebc40e28ebd597bea8ea2f4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/aaa28e139ebc40e28ebd597bea8ea2f4 2024-12-09T06:16:59,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/25881fb9a86f4dafa8f55df705fa1903 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/25881fb9a86f4dafa8f55df705fa1903 2024-12-09T06:16:59,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/04e065434d954f62867b95df54fc628f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/04e065434d954f62867b95df54fc628f 2024-12-09T06:16:59,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3555497bca594654aa34ace9cc5321c3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/3555497bca594654aa34ace9cc5321c3 2024-12-09T06:16:59,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/92a51bef401f4cc8ba8365b5b94edea1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/92a51bef401f4cc8ba8365b5b94edea1 2024-12-09T06:16:59,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/5e485af0897e415aa7af3c4d0f285bc7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/5e485af0897e415aa7af3c4d0f285bc7 2024-12-09T06:16:59,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b93a783a192c4ece915b7bae5d652300 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b93a783a192c4ece915b7bae5d652300 2024-12-09T06:16:59,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/bdf294b82f2f4c29a95f82f8ce62fed2 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/bdf294b82f2f4c29a95f82f8ce62fed2 2024-12-09T06:16:59,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/917b99381d6a4ef0a23d493b53cd7bc1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/917b99381d6a4ef0a23d493b53cd7bc1 2024-12-09T06:16:59,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b7fedd381c2847e48092ee2f26293c36 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/b7fedd381c2847e48092ee2f26293c36 2024-12-09T06:16:59,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/8243e3b4d7534d5391611b0ddd7964c8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/8243e3b4d7534d5391611b0ddd7964c8 2024-12-09T06:16:59,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/361266d2de3d41fa9f01d39a4b42f319 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/361266d2de3d41fa9f01d39a4b42f319 2024-12-09T06:16:59,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2bedc60da1ee457b8c7cb75ae352cc99 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/2bedc60da1ee457b8c7cb75ae352cc99 2024-12-09T06:16:59,944 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/recovered.edits/945.seqid, newMaxSeqId=945, maxSeqId=1 2024-12-09T06:16:59,946 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. 2024-12-09T06:16:59,947 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for f4594c92fbabea614394c10ba9359df8: Waiting for close lock at 1733725018552Running coprocessor pre-close hooks at 1733725018552Disabling compacts and flushes for region at 1733725018552Disabling writes for close at 1733725018552Obtaining lock to block concurrent updates at 1733725018552Preparing flush snapshotting stores in f4594c92fbabea614394c10ba9359df8 at 1733725018552Finished memstore snapshotting TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733725018553 (+1 ms)Flushing stores of TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8. at 1733725018553Flushing f4594c92fbabea614394c10ba9359df8/A: creating writer at 1733725018553Flushing f4594c92fbabea614394c10ba9359df8/A: appending metadata at 1733725018555 (+2 ms)Flushing f4594c92fbabea614394c10ba9359df8/A: closing flushed file at 1733725018555Flushing f4594c92fbabea614394c10ba9359df8/B: creating writer at 1733725018967 (+412 ms)Flushing f4594c92fbabea614394c10ba9359df8/B: appending metadata at 1733725018971 (+4 ms)Flushing f4594c92fbabea614394c10ba9359df8/B: closing flushed file at 1733725018971Flushing f4594c92fbabea614394c10ba9359df8/C: creating writer at 1733725019379 (+408 ms)Flushing f4594c92fbabea614394c10ba9359df8/C: appending metadata at 1733725019381 (+2 ms)Flushing f4594c92fbabea614394c10ba9359df8/C: closing flushed file at 1733725019382 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44edb7e9: reopening flushed file at 1733725019790 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e5fa199: reopening flushed file at 1733725019794 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64a16a12: reopening flushed file at 1733725019799 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f4594c92fbabea614394c10ba9359df8 in 1253ms, sequenceid=942, compaction requested=true at 1733725019805 (+6 ms)Writing region close event to WAL at 1733725019940 (+135 ms)Running coprocessor post-close hooks at 1733725019946 (+6 ms)Closed at 1733725019946 2024-12-09T06:16:59,949 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed f4594c92fbabea614394c10ba9359df8 2024-12-09T06:16:59,950 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=f4594c92fbabea614394c10ba9359df8, regionState=CLOSED 2024-12-09T06:16:59,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:16:59,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-09T06:16:59,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure f4594c92fbabea614394c10ba9359df8, server=82ac79f0323b,42797,1733724930590 in 1.5530 sec 2024-12-09T06:16:59,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-09T06:16:59,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f4594c92fbabea614394c10ba9359df8, UNASSIGN in 1.5590 sec 2024-12-09T06:16:59,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725019964"}]},"ts":"1733725019964"} 2024-12-09T06:16:59,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-09T06:16:59,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5640 sec 2024-12-09T06:16:59,966 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T06:16:59,966 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T06:16:59,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5760 sec 2024-12-09T06:17:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T06:17:00,537 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:00,538 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T06:17:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:00,540 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T06:17:00,540 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:00,543 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8 2024-12-09T06:17:00,545 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/recovered.edits] 2024-12-09T06:17:00,547 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cb27e7802dd436da1b83b5a10dcbd55 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/1cb27e7802dd436da1b83b5a10dcbd55 2024-12-09T06:17:00,548 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c994a5333e2a4548b2a2afe9064a3cd7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/c994a5333e2a4548b2a2afe9064a3cd7 2024-12-09T06:17:00,549 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/cd6f016ed1e84c108aa2b44d190e44f1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/A/cd6f016ed1e84c108aa2b44d190e44f1 2024-12-09T06:17:00,551 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/10cb893824cd4bcc8c3c11a03956cb1d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/10cb893824cd4bcc8c3c11a03956cb1d 2024-12-09T06:17:00,553 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2d1c9d5a244b4df98cde19ef1dc35243 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/2d1c9d5a244b4df98cde19ef1dc35243 2024-12-09T06:17:00,554 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/dbe9876461b147d491f4da78956e3fde to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/B/dbe9876461b147d491f4da78956e3fde 2024-12-09T06:17:00,556 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/13e0f9f4828f4b8a971942e9075f6006 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/13e0f9f4828f4b8a971942e9075f6006 2024-12-09T06:17:00,557 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/35bedef8796340c289d19cc5221a1e01 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/35bedef8796340c289d19cc5221a1e01 2024-12-09T06:17:00,559 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/884ad64dbb4b42aba0ee5b50a139245b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/884ad64dbb4b42aba0ee5b50a139245b 2024-12-09T06:17:00,560 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/c425034e74a2492890e4f7203971bc06 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/C/c425034e74a2492890e4f7203971bc06 2024-12-09T06:17:00,562 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/recovered.edits/945.seqid to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8/recovered.edits/945.seqid 2024-12-09T06:17:00,563 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/f4594c92fbabea614394c10ba9359df8 2024-12-09T06:17:00,563 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T06:17:00,565 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:00,568 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T06:17:00,570 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T06:17:00,571 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:00,571 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T06:17:00,572 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733725020571"}]},"ts":"9223372036854775807"} 2024-12-09T06:17:00,574 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T06:17:00,574 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f4594c92fbabea614394c10ba9359df8, NAME => 'TestAcidGuarantees,,1733724994167.f4594c92fbabea614394c10ba9359df8.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T06:17:00,574 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T06:17:00,574 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733725020574"}]},"ts":"9223372036854775807"} 2024-12-09T06:17:00,576 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T06:17:00,577 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:00,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-12-09T06:17:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T06:17:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T06:17:00,647 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:00,655 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_45612531_22 at /127.0.0.1:56030 [Waiting for operation #154] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_45612531_22 at /127.0.0.1:58598 [Waiting for operation #448] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-35193471_22 at /127.0.0.1:44308 [Waiting for operation #474] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1026 (was 1057), ProcessCount=11 (was 11), AvailableMemoryMB=4784 (was 4615) - AvailableMemoryMB LEAK? - 2024-12-09T06:17:00,664 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1026, ProcessCount=11, AvailableMemoryMB=4784 2024-12-09T06:17:00,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:17:00,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:17:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:00,667 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T06:17:00,667 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:00,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-12-09T06:17:00,668 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T06:17:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T06:17:00,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742136_1312 (size=1061) 2024-12-09T06:17:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T06:17:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T06:17:01,075 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d 2024-12-09T06:17:01,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742137_1313 (size=53) 2024-12-09T06:17:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T06:17:01,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:17:01,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1f2d999c565a1d93300e02727b70ded5, disabling compactions & flushes 2024-12-09T06:17:01,482 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:01,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:01,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. after waiting 0 ms 2024-12-09T06:17:01,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:01,482 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:01,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1f2d999c565a1d93300e02727b70ded5: Waiting for close lock at 1733725021482Disabling compacts and flushes for region at 1733725021482Disabling writes for close at 1733725021482Writing region close event to WAL at 1733725021482Closed at 1733725021482 2024-12-09T06:17:01,483 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T06:17:01,483 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733725021483"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733725021483"}]},"ts":"1733725021483"} 2024-12-09T06:17:01,485 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T06:17:01,486 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T06:17:01,486 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725021486"}]},"ts":"1733725021486"} 2024-12-09T06:17:01,488 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T06:17:01,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, ASSIGN}] 2024-12-09T06:17:01,494 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, ASSIGN 2024-12-09T06:17:01,495 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, ASSIGN; state=OFFLINE, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=false 2024-12-09T06:17:01,646 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=1f2d999c565a1d93300e02727b70ded5, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:01,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, ASSIGN because future has completed 2024-12-09T06:17:01,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:01,804 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:01,804 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:17:01,805 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,805 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:17:01,805 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,805 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T06:17:01,809 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,810 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:01,810 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2d999c565a1d93300e02727b70ded5 columnFamilyName A 2024-12-09T06:17:01,810 DEBUG [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:01,811 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(327): Store=1f2d999c565a1d93300e02727b70ded5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:01,811 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,817 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:01,818 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2d999c565a1d93300e02727b70ded5 columnFamilyName B 2024-12-09T06:17:01,818 DEBUG [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:01,818 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(327): Store=1f2d999c565a1d93300e02727b70ded5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:01,818 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,819 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:01,819 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2d999c565a1d93300e02727b70ded5 columnFamilyName C 2024-12-09T06:17:01,819 DEBUG [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:01,820 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(327): Store=1f2d999c565a1d93300e02727b70ded5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:01,820 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:01,820 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,821 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,821 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,822 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,822 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,823 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:17:01,824 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,826 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:17:01,826 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 1f2d999c565a1d93300e02727b70ded5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74454021, jitterRate=0.10945136845111847}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:17:01,826 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:01,827 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 1f2d999c565a1d93300e02727b70ded5: Running coprocessor pre-open hook at 1733725021805Writing region info on filesystem at 1733725021805Initializing all the Stores at 1733725021805Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725021806 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725021809 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725021809Cleaning up temporary data from old regions at 1733725021822 (+13 ms)Running coprocessor post-open hooks at 1733725021826 (+4 ms)Region opened successfully at 1733725021827 (+1 ms) 2024-12-09T06:17:01,828 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., pid=85, masterSystemTime=1733725021801 2024-12-09T06:17:01,830 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:01,830 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:01,830 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=1f2d999c565a1d93300e02727b70ded5, regionState=OPEN, openSeqNum=2, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:01,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:01,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-09T06:17:01,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 in 183 msec 2024-12-09T06:17:01,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-09T06:17:01,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, ASSIGN in 345 msec 2024-12-09T06:17:01,835 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T06:17:01,835 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725021835"}]},"ts":"1733725021835"} 2024-12-09T06:17:01,836 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T06:17:01,837 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T06:17:01,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-12-09T06:17:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T06:17:02,817 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:02,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:02,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:02,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:02,820 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbaa305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:02,822 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@299bc0a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:02,823 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:02,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:02,826 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T06:17:02,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 82ac79f0323b,43241,1733724929416 2024-12-09T06:17:02,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@421db79 2024-12-09T06:17:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T06:17:02,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T06:17:02,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:17:02,829 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:17:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:02,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742138_1314 (size=1097) 2024-12-09T06:17:03,243 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T06:17:03,243 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T06:17:03,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:17:03,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, REOPEN/MOVE}] 2024-12-09T06:17:03,253 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, REOPEN/MOVE 2024-12-09T06:17:03,254 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=1f2d999c565a1d93300e02727b70ded5, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:03,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, REOPEN/MOVE because future has completed 2024-12-09T06:17:03,256 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:17:03,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:03,408 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,408 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:17:03,409 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 1f2d999c565a1d93300e02727b70ded5, disabling compactions & flushes 2024-12-09T06:17:03,409 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,409 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,409 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. after waiting 0 ms 2024-12-09T06:17:03,409 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,412 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T06:17:03,412 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,412 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 1f2d999c565a1d93300e02727b70ded5: Waiting for close lock at 1733725023409Running coprocessor pre-close hooks at 1733725023409Disabling compacts and flushes for region at 1733725023409Disabling writes for close at 1733725023409Writing region close event to WAL at 1733725023409Running coprocessor post-close hooks at 1733725023412 (+3 ms)Closed at 1733725023412 2024-12-09T06:17:03,412 WARN [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 1f2d999c565a1d93300e02727b70ded5 to self. 2024-12-09T06:17:03,414 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,414 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=1f2d999c565a1d93300e02727b70ded5, regionState=CLOSED 2024-12-09T06:17:03,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:03,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-09T06:17:03,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 in 160 msec 2024-12-09T06:17:03,418 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, REOPEN/MOVE; state=CLOSED, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=true 2024-12-09T06:17:03,569 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=1f2d999c565a1d93300e02727b70ded5, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:03,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, REOPEN/MOVE because future has completed 2024-12-09T06:17:03,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:03,726 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,726 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:17:03,727 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,727 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:17:03,727 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,727 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,728 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,729 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:03,730 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2d999c565a1d93300e02727b70ded5 columnFamilyName A 2024-12-09T06:17:03,731 DEBUG [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:03,732 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(327): Store=1f2d999c565a1d93300e02727b70ded5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:03,733 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,734 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:03,734 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2d999c565a1d93300e02727b70ded5 columnFamilyName B 2024-12-09T06:17:03,734 DEBUG [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:03,735 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(327): Store=1f2d999c565a1d93300e02727b70ded5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:03,735 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,736 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:03,736 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2d999c565a1d93300e02727b70ded5 columnFamilyName C 2024-12-09T06:17:03,736 DEBUG [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:03,737 INFO [StoreOpener-1f2d999c565a1d93300e02727b70ded5-1 {}] regionserver.HStore(327): Store=1f2d999c565a1d93300e02727b70ded5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:03,737 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,737 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,737 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,738 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,739 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,739 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,740 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:17:03,742 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,742 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 1f2d999c565a1d93300e02727b70ded5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60441594, jitterRate=-0.09935006499290466}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:17:03,743 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,744 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 1f2d999c565a1d93300e02727b70ded5: Running coprocessor pre-open hook at 1733725023727Writing region info on filesystem at 1733725023727Initializing all the Stores at 1733725023728 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725023728Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725023728Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725023728Cleaning up temporary data from old regions at 1733725023739 (+11 ms)Running coprocessor post-open hooks at 1733725023743 (+4 ms)Region opened successfully at 1733725023743 2024-12-09T06:17:03,744 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., pid=90, masterSystemTime=1733725023723 2024-12-09T06:17:03,746 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,746 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,746 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=1f2d999c565a1d93300e02727b70ded5, regionState=OPEN, openSeqNum=5, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:03,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:03,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-12-09T06:17:03,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 in 177 msec 2024-12-09T06:17:03,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-09T06:17:03,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, REOPEN/MOVE in 499 msec 2024-12-09T06:17:03,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-09T06:17:03,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-12-09T06:17:03,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-12-09T06:17:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T06:17:03,767 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:03,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,769 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2504834e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,771 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2849228b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,774 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71dbfcc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,775 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@269ee678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,777 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3394e08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,779 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e7acc0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,781 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4970a9d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,782 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3688c186, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,784 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6103d92e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,785 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da9b687, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,787 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@709b54e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,789 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1e8af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,791 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f143c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,793 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78000323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,795 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58009272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e120f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,800 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244d7102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e03556, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:03,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:03,804 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9cc09d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:03,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5177918e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:03,807 DEBUG [Thread-1441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,807 DEBUG [Thread-1445 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,807 DEBUG [Thread-1460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,807 DEBUG [Thread-1454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,808 DEBUG [Thread-1458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,808 DEBUG [Thread-1449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,809 DEBUG [Thread-1456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,810 DEBUG [Thread-1452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-09T06:17:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T06:17:03,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:03,813 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:03,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:03,813 DEBUG [Thread-1443 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,813 DEBUG [Thread-1447 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:03,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:03,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:17:03,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:03,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:03,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:03,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:03,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:03,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:03,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092f0a10d6320c430d94087f6f0273d95f_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725023826/Put/seqid=0 2024-12-09T06:17:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725083856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725083856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725083857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725083862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725083869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:03,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725023852, max timestamp=1733725023867Num uniques 859; 2024-12-09T06:17:03,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:03,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725023853, max timestamp=1733725023867Num uniques 859; 2024-12-09T06:17:03,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:17:03,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725023853, max timestamp=1733725023867Num uniques 859; 2024-12-09T06:17:03,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:17:03,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742139_1315 (size=14594) 2024-12-09T06:17:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T06:17:03,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:03,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:03,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:03,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:03,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725084005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725084007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725084009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725084011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725084011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T06:17:04,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725084121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725084127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725084127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725084128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725084128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:04,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:04,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:04,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092f0a10d6320c430d94087f6f0273d95f_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f0a10d6320c430d94087f6f0273d95f_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:04,291 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/8623b8d1acdc42879f486a8730ea1124, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:04,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/8623b8d1acdc42879f486a8730ea1124 is 175, key is test_row_0/A:col10/1733725023826/Put/seqid=0 2024-12-09T06:17:04,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742140_1316 (size=39549) 2024-12-09T06:17:04,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725084338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725084347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725084347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725084347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725084349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T06:17:04,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725084657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725084657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725084657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725084657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725084667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:04,709 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/8623b8d1acdc42879f486a8730ea1124 2024-12-09T06:17:04,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c27e65fd30644a6d88c3c4beafe72663 is 50, key is test_row_0/B:col10/1733725023826/Put/seqid=0 2024-12-09T06:17:04,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742141_1317 (size=12001) 2024-12-09T06:17:04,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c27e65fd30644a6d88c3c4beafe72663 2024-12-09T06:17:04,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/e5168e7a9e624db796f86dd1e87a42f3 is 50, key is test_row_0/C:col10/1733725023826/Put/seqid=0 2024-12-09T06:17:04,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742142_1318 (size=12001) 2024-12-09T06:17:04,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:04,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T06:17:05,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:05,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:05,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:05,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:05,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:05,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:05,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/e5168e7a9e624db796f86dd1e87a42f3 2024-12-09T06:17:05,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725085176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:05,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725085176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/8623b8d1acdc42879f486a8730ea1124 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/8623b8d1acdc42879f486a8730ea1124 2024-12-09T06:17:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725085176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:05,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/8623b8d1acdc42879f486a8730ea1124, entries=200, sequenceid=15, filesize=38.6 K 2024-12-09T06:17:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c27e65fd30644a6d88c3c4beafe72663 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c27e65fd30644a6d88c3c4beafe72663 2024-12-09T06:17:05,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c27e65fd30644a6d88c3c4beafe72663, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T06:17:05,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/e5168e7a9e624db796f86dd1e87a42f3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e5168e7a9e624db796f86dd1e87a42f3 2024-12-09T06:17:05,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725085187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:05,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725085187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:05,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:05,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:05,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:05,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:05,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:05,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:05,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e5168e7a9e624db796f86dd1e87a42f3, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T06:17:05,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=135.63 KB/138880 for 1f2d999c565a1d93300e02727b70ded5 in 1374ms, sequenceid=15, compaction requested=false 2024-12-09T06:17:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:05,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T06:17:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:05,353 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-12-09T06:17:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:05,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120995b832f62f0c41c0bc4815d93ec3271c_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725023994/Put/seqid=0 2024-12-09T06:17:05,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742143_1319 (size=12154) 2024-12-09T06:17:05,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,419 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120995b832f62f0c41c0bc4815d93ec3271c_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120995b832f62f0c41c0bc4815d93ec3271c_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:05,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/80c7d11d69cc456bb9600dbca28dc35b, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/80c7d11d69cc456bb9600dbca28dc35b is 175, key is test_row_0/A:col10/1733725023994/Put/seqid=0 2024-12-09T06:17:05,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742144_1320 (size=30955) 2024-12-09T06:17:05,461 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/80c7d11d69cc456bb9600dbca28dc35b 2024-12-09T06:17:05,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/902946ee822e40a88348434c0293ef3b is 50, key is test_row_0/B:col10/1733725023994/Put/seqid=0 2024-12-09T06:17:05,469 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T06:17:05,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742145_1321 (size=12001) 2024-12-09T06:17:05,505 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/902946ee822e40a88348434c0293ef3b 2024-12-09T06:17:05,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/dcd0fe80d8dd429ba154f7885b7ba895 is 50, key is test_row_0/C:col10/1733725023994/Put/seqid=0 2024-12-09T06:17:05,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742146_1322 (size=12001) 2024-12-09T06:17:05,526 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/dcd0fe80d8dd429ba154f7885b7ba895 2024-12-09T06:17:05,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/80c7d11d69cc456bb9600dbca28dc35b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/80c7d11d69cc456bb9600dbca28dc35b 2024-12-09T06:17:05,543 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/80c7d11d69cc456bb9600dbca28dc35b, entries=150, sequenceid=52, filesize=30.2 K 2024-12-09T06:17:05,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/902946ee822e40a88348434c0293ef3b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/902946ee822e40a88348434c0293ef3b 2024-12-09T06:17:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,549 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/902946ee822e40a88348434c0293ef3b, entries=150, sequenceid=52, filesize=11.7 K 2024-12-09T06:17:05,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/dcd0fe80d8dd429ba154f7885b7ba895 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/dcd0fe80d8dd429ba154f7885b7ba895 2024-12-09T06:17:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,554 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/dcd0fe80d8dd429ba154f7885b7ba895, entries=150, sequenceid=52, filesize=11.7 K 2024-12-09T06:17:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,554 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~135.63 KB/138880, heapSize ~388.19 KB/397504, currentSize=0 B/0 for 1f2d999c565a1d93300e02727b70ded5 in 201ms, sequenceid=52, compaction requested=false 2024-12-09T06:17:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-09T06:17:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-09T06:17:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-09T06:17:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7430 sec 2024-12-09T06:17:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.7510 sec 2024-12-09T06:17:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T06:17:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,957 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,957 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-09T06:17:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T06:17:05,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,960 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:05,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T06:17:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T06:17:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-09T06:17:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-09T06:17:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-09T06:17:06,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-12-09T06:17:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 168 msec 2024-12-09T06:17:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T06:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,277 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,277 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-09T06:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T06:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,279 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T06:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:06,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:06,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:06,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:06,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097790059db16b409ba007d61756bb9219_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725026385/Put/seqid=0 2024-12-09T06:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T06:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742147_1323 (size=12154) 2024-12-09T06:17:06,440 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,444 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097790059db16b409ba007d61756bb9219_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097790059db16b409ba007d61756bb9219_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,446 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/c0c8ad172659400190e8a961c8db2254, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:06,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/c0c8ad172659400190e8a961c8db2254 is 175, key is test_row_0/A:col10/1733725026385/Put/seqid=0 2024-12-09T06:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:06,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742148_1324 (size=30951) 2024-12-09T06:17:06,473 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/c0c8ad172659400190e8a961c8db2254 2024-12-09T06:17:06,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/2e633ba42a2a4d9dbf81ac2fbc452c79 is 50, key is test_row_0/B:col10/1733725026385/Put/seqid=0 2024-12-09T06:17:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725026445, max timestamp=1733725026483Num uniques 859; 2024-12-09T06:17:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725026445, max timestamp=1733725026483Num uniques 859; 2024-12-09T06:17:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:06,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725026445, max timestamp=1733725026483Num uniques 859; 2024-12-09T06:17:06,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:06,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742149_1325 (size=9657) 2024-12-09T06:17:06,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725086514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725086514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725086515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725086516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725086522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T06:17:06,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:06,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T06:17:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725086626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725086626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725086626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725086626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725086626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T06:17:06,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:06,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725086836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725086836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725086837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725086837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725086837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:06,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T06:17:06,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:06,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:06,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:06,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/2e633ba42a2a4d9dbf81ac2fbc452c79 2024-12-09T06:17:06,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/e3df8cb8f8844c51a2f07a780d0e4670 is 50, key is test_row_0/C:col10/1733725026385/Put/seqid=0 2024-12-09T06:17:06,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742150_1326 (size=9657) 2024-12-09T06:17:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T06:17:07,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T06:17:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:07,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725087146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725087146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725087146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725087146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725087157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T06:17:07,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:07,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:07,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:07,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/e3df8cb8f8844c51a2f07a780d0e4670 2024-12-09T06:17:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/c0c8ad172659400190e8a961c8db2254 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0c8ad172659400190e8a961c8db2254 2024-12-09T06:17:07,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0c8ad172659400190e8a961c8db2254, entries=150, sequenceid=63, filesize=30.2 K 2024-12-09T06:17:07,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/2e633ba42a2a4d9dbf81ac2fbc452c79 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/2e633ba42a2a4d9dbf81ac2fbc452c79 2024-12-09T06:17:07,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/2e633ba42a2a4d9dbf81ac2fbc452c79, entries=100, sequenceid=63, filesize=9.4 K 2024-12-09T06:17:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/e3df8cb8f8844c51a2f07a780d0e4670 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e3df8cb8f8844c51a2f07a780d0e4670 2024-12-09T06:17:07,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e3df8cb8f8844c51a2f07a780d0e4670, entries=100, sequenceid=63, filesize=9.4 K 2024-12-09T06:17:07,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1f2d999c565a1d93300e02727b70ded5 in 919ms, sequenceid=63, compaction requested=true 2024-12-09T06:17:07,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:07,320 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:07,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:07,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:07,320 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:07,320 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:07,320 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/A is initiating minor compaction (all files) 2024-12-09T06:17:07,321 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/A in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:07,321 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/8623b8d1acdc42879f486a8730ea1124, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/80c7d11d69cc456bb9600dbca28dc35b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0c8ad172659400190e8a961c8db2254] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=99.1 K 2024-12-09T06:17:07,321 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,321 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/8623b8d1acdc42879f486a8730ea1124, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/80c7d11d69cc456bb9600dbca28dc35b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0c8ad172659400190e8a961c8db2254] 2024-12-09T06:17:07,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:07,321 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:07,321 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/B is initiating minor compaction (all files) 2024-12-09T06:17:07,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:07,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:07,321 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/B in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,321 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c27e65fd30644a6d88c3c4beafe72663, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/902946ee822e40a88348434c0293ef3b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/2e633ba42a2a4d9dbf81ac2fbc452c79] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=32.9 K 2024-12-09T06:17:07,321 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8623b8d1acdc42879f486a8730ea1124, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733725023821 2024-12-09T06:17:07,322 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c27e65fd30644a6d88c3c4beafe72663, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733725023823 2024-12-09T06:17:07,322 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80c7d11d69cc456bb9600dbca28dc35b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733725023994 2024-12-09T06:17:07,322 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 902946ee822e40a88348434c0293ef3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733725023994 2024-12-09T06:17:07,322 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c8ad172659400190e8a961c8db2254, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733725026340 2024-12-09T06:17:07,322 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e633ba42a2a4d9dbf81ac2fbc452c79, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733725026385 2024-12-09T06:17:07,327 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:07,328 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:07,329 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/a4783ff8893847609c7af96f2a422dd6 is 50, key is test_row_0/B:col10/1733725026385/Put/seqid=0 2024-12-09T06:17:07,331 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209b10747f2a8a04c3ca4b20c4ea0ec3e5f_1f2d999c565a1d93300e02727b70ded5 store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:07,332 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209b10747f2a8a04c3ca4b20c4ea0ec3e5f_1f2d999c565a1d93300e02727b70ded5, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:07,333 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b10747f2a8a04c3ca4b20c4ea0ec3e5f_1f2d999c565a1d93300e02727b70ded5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:07,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742151_1327 (size=12104) 2024-12-09T06:17:07,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742152_1328 (size=4469) 2024-12-09T06:17:07,340 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/a4783ff8893847609c7af96f2a422dd6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a4783ff8893847609c7af96f2a422dd6 2024-12-09T06:17:07,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T06:17:07,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,352 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-09T06:17:07,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:07,354 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/B of 1f2d999c565a1d93300e02727b70ded5 into a4783ff8893847609c7af96f2a422dd6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:07,354 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:07,354 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/B, priority=13, startTime=1733725027320; duration=0sec 2024-12-09T06:17:07,354 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:07,354 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:B 2024-12-09T06:17:07,354 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:07,355 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:07,355 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/C is initiating minor compaction (all files) 2024-12-09T06:17:07,355 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/C in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,355 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e5168e7a9e624db796f86dd1e87a42f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/dcd0fe80d8dd429ba154f7885b7ba895, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e3df8cb8f8844c51a2f07a780d0e4670] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=32.9 K 2024-12-09T06:17:07,356 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e5168e7a9e624db796f86dd1e87a42f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733725023823 2024-12-09T06:17:07,357 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting dcd0fe80d8dd429ba154f7885b7ba895, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733725023994 2024-12-09T06:17:07,357 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e3df8cb8f8844c51a2f07a780d0e4670, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733725026385 2024-12-09T06:17:07,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e4c85469d818428aa3fab4174458db9e_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725026513/Put/seqid=0 2024-12-09T06:17:07,370 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:07,371 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/459a8af9731b4372b8f805979e5c40f9 is 50, key is test_row_0/C:col10/1733725026385/Put/seqid=0 2024-12-09T06:17:07,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742153_1329 (size=12154) 2024-12-09T06:17:07,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:07,379 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e4c85469d818428aa3fab4174458db9e_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e4c85469d818428aa3fab4174458db9e_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:07,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/6a810fb14d744904897bd1e618783a90, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:07,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/6a810fb14d744904897bd1e618783a90 is 175, key is test_row_0/A:col10/1733725026513/Put/seqid=0 2024-12-09T06:17:07,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742154_1330 (size=12104) 2024-12-09T06:17:07,394 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/459a8af9731b4372b8f805979e5c40f9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/459a8af9731b4372b8f805979e5c40f9 2024-12-09T06:17:07,398 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/C of 1f2d999c565a1d93300e02727b70ded5 into 459a8af9731b4372b8f805979e5c40f9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:07,398 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:07,398 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/C, priority=13, startTime=1733725027321; duration=0sec 2024-12-09T06:17:07,398 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:07,398 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:C 2024-12-09T06:17:07,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742155_1331 (size=30955) 2024-12-09T06:17:07,402 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/6a810fb14d744904897bd1e618783a90 2024-12-09T06:17:07,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c15d9c196281440f88204d9d100126e8 is 50, key is test_row_0/B:col10/1733725026513/Put/seqid=0 2024-12-09T06:17:07,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742156_1332 (size=12001) 2024-12-09T06:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T06:17:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:07,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:07,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725087676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725087676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725087677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725087678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725087679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,740 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#A#compaction#283 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:07,741 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/e1258c50f5824afa8ebfb331cb1ce178 is 175, key is test_row_0/A:col10/1733725026385/Put/seqid=0 2024-12-09T06:17:07,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742157_1333 (size=31165) 2024-12-09T06:17:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725087798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725087798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725087798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725087799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:07,815 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c15d9c196281440f88204d9d100126e8 2024-12-09T06:17:07,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/91971c978c3f4e348a5700033491bdee is 50, key is test_row_0/C:col10/1733725026513/Put/seqid=0 2024-12-09T06:17:07,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742158_1334 (size=12001) 2024-12-09T06:17:07,833 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/91971c978c3f4e348a5700033491bdee 2024-12-09T06:17:07,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/6a810fb14d744904897bd1e618783a90 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/6a810fb14d744904897bd1e618783a90 2024-12-09T06:17:07,843 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/6a810fb14d744904897bd1e618783a90, entries=150, sequenceid=98, filesize=30.2 K 2024-12-09T06:17:07,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c15d9c196281440f88204d9d100126e8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c15d9c196281440f88204d9d100126e8 2024-12-09T06:17:07,847 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c15d9c196281440f88204d9d100126e8, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T06:17:07,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/91971c978c3f4e348a5700033491bdee as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/91971c978c3f4e348a5700033491bdee 2024-12-09T06:17:07,852 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/91971c978c3f4e348a5700033491bdee, entries=150, sequenceid=98, filesize=11.7 K 2024-12-09T06:17:07,853 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 1f2d999c565a1d93300e02727b70ded5 in 501ms, sequenceid=98, compaction requested=false 2024-12-09T06:17:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-09T06:17:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-09T06:17:07,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-09T06:17:07,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-12-09T06:17:07,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.5800 sec 2024-12-09T06:17:08,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:17:08,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:08,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:08,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:08,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:08,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:08,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:08,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120904c77a0a99804263b72e8b13cad15bec_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725027675/Put/seqid=0 2024-12-09T06:17:08,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742159_1335 (size=14594) 2024-12-09T06:17:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725088078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725088080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725088082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725028052, max timestamp=1733725028094Num uniques 859; 2024-12-09T06:17:08,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:08,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725028052, max timestamp=1733725028094Num uniques 859; 2024-12-09T06:17:08,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:08,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725028052, max timestamp=1733725028094Num uniques 859; 2024-12-09T06:17:08,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:08,151 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/e1258c50f5824afa8ebfb331cb1ce178 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e1258c50f5824afa8ebfb331cb1ce178 2024-12-09T06:17:08,158 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/A of 1f2d999c565a1d93300e02727b70ded5 into e1258c50f5824afa8ebfb331cb1ce178(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:08,158 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:08,159 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/A, priority=13, startTime=1733725027319; duration=0sec 2024-12-09T06:17:08,159 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:08,159 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:A 2024-12-09T06:17:08,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725088232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725088237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725088242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725088246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725088349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725088356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725088367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725088367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T06:17:08,427 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:08,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-09T06:17:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T06:17:08,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:08,434 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:08,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:08,450 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:08,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120904c77a0a99804263b72e8b13cad15bec_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120904c77a0a99804263b72e8b13cad15bec_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:08,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/79c017bdcb35432fa4936353c78a1a31, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:08,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/79c017bdcb35432fa4936353c78a1a31 is 175, key is test_row_0/A:col10/1733725027675/Put/seqid=0 2024-12-09T06:17:08,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742160_1336 (size=39549) 2024-12-09T06:17:08,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/79c017bdcb35432fa4936353c78a1a31 2024-12-09T06:17:08,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/ea0d3dd61da44d799b91f8e820735a75 is 50, key is test_row_0/B:col10/1733725027675/Put/seqid=0 2024-12-09T06:17:08,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742161_1337 (size=12001) 2024-12-09T06:17:08,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/ea0d3dd61da44d799b91f8e820735a75 2024-12-09T06:17:08,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/98fd306416d0435c88833d8089867d89 is 50, key is test_row_0/C:col10/1733725027675/Put/seqid=0 2024-12-09T06:17:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T06:17:08,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742162_1338 (size=12001) 2024-12-09T06:17:08,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725088567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725088576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T06:17:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:08,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725088587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725088587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725088697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,708 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 1f2d999c565a1d93300e02727b70ded5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T06:17:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T06:17:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725088886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725088886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T06:17:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:08,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:08,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725088907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725088907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:08,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:08,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/98fd306416d0435c88833d8089867d89 2024-12-09T06:17:08,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/79c017bdcb35432fa4936353c78a1a31 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/79c017bdcb35432fa4936353c78a1a31 2024-12-09T06:17:08,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/79c017bdcb35432fa4936353c78a1a31, entries=200, sequenceid=111, filesize=38.6 K 2024-12-09T06:17:08,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/ea0d3dd61da44d799b91f8e820735a75 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ea0d3dd61da44d799b91f8e820735a75 2024-12-09T06:17:08,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ea0d3dd61da44d799b91f8e820735a75, entries=150, sequenceid=111, filesize=11.7 K 2024-12-09T06:17:08,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/98fd306416d0435c88833d8089867d89 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/98fd306416d0435c88833d8089867d89 2024-12-09T06:17:08,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/98fd306416d0435c88833d8089867d89, entries=150, sequenceid=111, filesize=11.7 K 2024-12-09T06:17:08,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1f2d999c565a1d93300e02727b70ded5 in 961ms, sequenceid=111, compaction requested=true 2024-12-09T06:17:08,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:08,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:08,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:08,973 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:08,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:08,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:08,973 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:08,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:08,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:08,974 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:08,974 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/B is initiating minor compaction (all files) 2024-12-09T06:17:08,974 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/B in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,974 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a4783ff8893847609c7af96f2a422dd6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c15d9c196281440f88204d9d100126e8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ea0d3dd61da44d799b91f8e820735a75] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=35.3 K 2024-12-09T06:17:08,974 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:08,975 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/A is initiating minor compaction (all files) 2024-12-09T06:17:08,975 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/A in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,975 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e1258c50f5824afa8ebfb331cb1ce178, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/6a810fb14d744904897bd1e618783a90, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/79c017bdcb35432fa4936353c78a1a31] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=99.3 K 2024-12-09T06:17:08,975 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:08,975 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e1258c50f5824afa8ebfb331cb1ce178, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/6a810fb14d744904897bd1e618783a90, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/79c017bdcb35432fa4936353c78a1a31] 2024-12-09T06:17:08,975 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a4783ff8893847609c7af96f2a422dd6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733725024008 2024-12-09T06:17:08,975 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1258c50f5824afa8ebfb331cb1ce178, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733725024008 2024-12-09T06:17:08,975 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c15d9c196281440f88204d9d100126e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733725026511 2024-12-09T06:17:08,976 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a810fb14d744904897bd1e618783a90, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733725026511 2024-12-09T06:17:08,976 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0d3dd61da44d799b91f8e820735a75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733725027675 2024-12-09T06:17:08,976 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79c017bdcb35432fa4936353c78a1a31, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733725027675 2024-12-09T06:17:08,995 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:09,007 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#B#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:09,007 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/46fb74ab242f4cc58c10cb41faa55ccb is 50, key is test_row_0/B:col10/1733725027675/Put/seqid=0 2024-12-09T06:17:09,010 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209b74580da70224fef9f279e0683614683_1f2d999c565a1d93300e02727b70ded5 store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:09,012 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209b74580da70224fef9f279e0683614683_1f2d999c565a1d93300e02727b70ded5, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:09,012 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b74580da70224fef9f279e0683614683_1f2d999c565a1d93300e02727b70ded5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:09,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742163_1339 (size=12207) 2024-12-09T06:17:09,040 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/46fb74ab242f4cc58c10cb41faa55ccb as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/46fb74ab242f4cc58c10cb41faa55ccb 2024-12-09T06:17:09,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T06:17:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:09,049 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T06:17:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:09,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742164_1340 (size=4469) 2024-12-09T06:17:09,052 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/B of 1f2d999c565a1d93300e02727b70ded5 into 46fb74ab242f4cc58c10cb41faa55ccb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:09,052 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:09,052 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/B, priority=13, startTime=1733725028973; duration=0sec 2024-12-09T06:17:09,053 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:09,053 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:B 2024-12-09T06:17:09,053 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:09,054 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#A#compaction#291 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:09,054 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/049a0a403c174ab29db4f77759908a3f is 175, key is test_row_0/A:col10/1733725027675/Put/seqid=0 2024-12-09T06:17:09,054 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:09,055 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/C is initiating minor compaction (all files) 2024-12-09T06:17:09,055 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/C in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:09,055 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/459a8af9731b4372b8f805979e5c40f9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/91971c978c3f4e348a5700033491bdee, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/98fd306416d0435c88833d8089867d89] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=35.3 K 2024-12-09T06:17:09,055 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 459a8af9731b4372b8f805979e5c40f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733725024008 2024-12-09T06:17:09,056 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 91971c978c3f4e348a5700033491bdee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733725026511 2024-12-09T06:17:09,056 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 98fd306416d0435c88833d8089867d89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733725027675 2024-12-09T06:17:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T06:17:09,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090efad58528374c4289f19cd12aefc450_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725028236/Put/seqid=0 2024-12-09T06:17:09,079 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:09,080 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/3f5054c3eee64bc3898ea320f3748968 is 50, key is test_row_0/C:col10/1733725027675/Put/seqid=0 2024-12-09T06:17:09,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742165_1341 (size=31161) 2024-12-09T06:17:09,095 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/049a0a403c174ab29db4f77759908a3f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/049a0a403c174ab29db4f77759908a3f 2024-12-09T06:17:09,108 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/A of 1f2d999c565a1d93300e02727b70ded5 into 049a0a403c174ab29db4f77759908a3f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:09,108 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:09,108 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/A, priority=13, startTime=1733725028973; duration=0sec 2024-12-09T06:17:09,109 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:09,109 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:A 2024-12-09T06:17:09,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742167_1343 (size=12207) 2024-12-09T06:17:09,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742166_1342 (size=12304) 2024-12-09T06:17:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:09,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725089417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725089420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725089421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725089425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,533 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/3f5054c3eee64bc3898ea320f3748968 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/3f5054c3eee64bc3898ea320f3748968 2024-12-09T06:17:09,537 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/C of 1f2d999c565a1d93300e02727b70ded5 into 3f5054c3eee64bc3898ea320f3748968(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:09,537 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:09,537 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/C, priority=13, startTime=1733725028973; duration=0sec 2024-12-09T06:17:09,537 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:09,537 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:C 2024-12-09T06:17:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725089536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725089536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725089536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,541 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090efad58528374c4289f19cd12aefc450_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090efad58528374c4289f19cd12aefc450_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/5366220415534be28f14d3cca652c625, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/5366220415534be28f14d3cca652c625 is 175, key is test_row_0/A:col10/1733725028236/Put/seqid=0 2024-12-09T06:17:09,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742168_1344 (size=31105) 2024-12-09T06:17:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T06:17:09,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725089747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725089747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725089749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:09,946 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/5366220415534be28f14d3cca652c625 2024-12-09T06:17:09,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/98ddd7d75afd4a62853c5ac32d0663af is 50, key is test_row_0/B:col10/1733725028236/Put/seqid=0 2024-12-09T06:17:09,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742169_1345 (size=12151) 2024-12-09T06:17:10,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725090056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:10,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725090066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:10,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725090067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:10,357 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/98ddd7d75afd4a62853c5ac32d0663af 2024-12-09T06:17:10,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/fa5e349adc6e4159b92e3ad33a6355d1 is 50, key is test_row_0/C:col10/1733725028236/Put/seqid=0 2024-12-09T06:17:10,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742170_1346 (size=12151) 2024-12-09T06:17:10,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725090436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:10,442 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 1f2d999c565a1d93300e02727b70ded5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:10,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725090576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:10,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725090576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725090577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T06:17:10,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725090727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:10,735 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 1f2d999c565a1d93300e02727b70ded5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:10,771 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/fa5e349adc6e4159b92e3ad33a6355d1 2024-12-09T06:17:10,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/5366220415534be28f14d3cca652c625 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5366220415534be28f14d3cca652c625 2024-12-09T06:17:10,779 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5366220415534be28f14d3cca652c625, entries=150, sequenceid=149, filesize=30.4 K 2024-12-09T06:17:10,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/98ddd7d75afd4a62853c5ac32d0663af as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/98ddd7d75afd4a62853c5ac32d0663af 2024-12-09T06:17:10,784 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/98ddd7d75afd4a62853c5ac32d0663af, entries=150, sequenceid=149, filesize=11.9 K 2024-12-09T06:17:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/fa5e349adc6e4159b92e3ad33a6355d1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fa5e349adc6e4159b92e3ad33a6355d1 2024-12-09T06:17:10,789 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fa5e349adc6e4159b92e3ad33a6355d1, entries=150, sequenceid=149, filesize=11.9 K 2024-12-09T06:17:10,791 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 1f2d999c565a1d93300e02727b70ded5 in 1742ms, sequenceid=149, compaction requested=false 2024-12-09T06:17:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-09T06:17:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-09T06:17:10,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-09T06:17:10,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3590 sec 2024-12-09T06:17:10,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.3670 sec 2024-12-09T06:17:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:11,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:17:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:11,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120957a4c6319c3842218e74ddbb0e80ee79_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725029424/Put/seqid=0 2024-12-09T06:17:11,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742171_1347 (size=12304) 2024-12-09T06:17:11,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725031626, max timestamp=1733725031641Num uniques 859; 2024-12-09T06:17:11,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725031626, max timestamp=1733725031641Num uniques 859; 2024-12-09T06:17:11,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:11,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725031626, max timestamp=1733725031641Num uniques 859; 2024-12-09T06:17:11,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:11,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725091692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725091692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:11,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725091693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:11,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725091806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:11,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725091807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:11,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725091817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,012 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:12,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120957a4c6319c3842218e74ddbb0e80ee79_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120957a4c6319c3842218e74ddbb0e80ee79_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:12,017 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/0666f491e76f45c49abb554c2861148d, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:12,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/0666f491e76f45c49abb554c2861148d is 175, key is test_row_0/A:col10/1733725029424/Put/seqid=0 2024-12-09T06:17:12,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742172_1348 (size=31105) 2024-12-09T06:17:12,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/0666f491e76f45c49abb554c2861148d 2024-12-09T06:17:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725092016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725092016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/12d6d215fb364af3963596b4003721a0 is 50, key is test_row_0/B:col10/1733725029424/Put/seqid=0 2024-12-09T06:17:12,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742173_1349 (size=12151) 2024-12-09T06:17:12,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725092027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725092337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725092339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725092346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/12d6d215fb364af3963596b4003721a0 2024-12-09T06:17:12,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/8a67f44eadc243c19208eeefc0d42b8b is 50, key is test_row_0/C:col10/1733725029424/Put/seqid=0 2024-12-09T06:17:12,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742174_1350 (size=12151) 2024-12-09T06:17:12,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725092466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,471 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 1f2d999c565a1d93300e02727b70ded5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T06:17:12,606 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:12,607 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-09T06:17:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T06:17:12,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:12,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:12,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T06:17:12,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T06:17:12,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:12,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:12,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:12,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:12,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:12,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/8a67f44eadc243c19208eeefc0d42b8b 2024-12-09T06:17:12,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725092846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/0666f491e76f45c49abb554c2861148d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0666f491e76f45c49abb554c2861148d 2024-12-09T06:17:12,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0666f491e76f45c49abb554c2861148d, entries=150, sequenceid=163, filesize=30.4 K 2024-12-09T06:17:12,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/12d6d215fb364af3963596b4003721a0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/12d6d215fb364af3963596b4003721a0 2024-12-09T06:17:12,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/12d6d215fb364af3963596b4003721a0, entries=150, sequenceid=163, filesize=11.9 K 2024-12-09T06:17:12,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/8a67f44eadc243c19208eeefc0d42b8b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/8a67f44eadc243c19208eeefc0d42b8b 2024-12-09T06:17:12,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725092857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/8a67f44eadc243c19208eeefc0d42b8b, entries=150, sequenceid=163, filesize=11.9 K 2024-12-09T06:17:12,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1f2d999c565a1d93300e02727b70ded5 in 1280ms, sequenceid=163, compaction requested=true 2024-12-09T06:17:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:12,870 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:12,870 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:12,871 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:12,871 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:12,871 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/A is initiating minor compaction (all files) 2024-12-09T06:17:12,871 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/B is initiating minor compaction (all files) 2024-12-09T06:17:12,871 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/B in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:12,871 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/A in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:12,871 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/46fb74ab242f4cc58c10cb41faa55ccb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/98ddd7d75afd4a62853c5ac32d0663af, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/12d6d215fb364af3963596b4003721a0] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=35.7 K 2024-12-09T06:17:12,871 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/049a0a403c174ab29db4f77759908a3f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5366220415534be28f14d3cca652c625, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0666f491e76f45c49abb554c2861148d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=91.2 K 2024-12-09T06:17:12,871 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:12,871 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/049a0a403c174ab29db4f77759908a3f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5366220415534be28f14d3cca652c625, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0666f491e76f45c49abb554c2861148d] 2024-12-09T06:17:12,872 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 46fb74ab242f4cc58c10cb41faa55ccb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733725027675 2024-12-09T06:17:12,872 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 049a0a403c174ab29db4f77759908a3f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733725027675 2024-12-09T06:17:12,872 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ddd7d75afd4a62853c5ac32d0663af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733725028219 2024-12-09T06:17:12,872 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5366220415534be28f14d3cca652c625, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733725028219 2024-12-09T06:17:12,872 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d6d215fb364af3963596b4003721a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733725029420 2024-12-09T06:17:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:12,873 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0666f491e76f45c49abb554c2861148d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733725029420 2024-12-09T06:17:12,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T06:17:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:12,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:12,885 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:12,886 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209e4b2f4edec274827af672881fef60d51_1f2d999c565a1d93300e02727b70ded5 store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:12,887 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209e4b2f4edec274827af672881fef60d51_1f2d999c565a1d93300e02727b70ded5, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:12,888 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e4b2f4edec274827af672881fef60d51_1f2d999c565a1d93300e02727b70ded5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:12,893 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#B#compaction#301 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:12,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099ab74533999342a0a905a4c636627b3b_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725031681/Put/seqid=0 2024-12-09T06:17:12,896 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/3fc126140eae47978bbc600a8a16a36e is 50, key is test_row_0/B:col10/1733725029424/Put/seqid=0 2024-12-09T06:17:12,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T06:17:12,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:12,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:12,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:12,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:12,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:12,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742175_1351 (size=4469) 2024-12-09T06:17:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:12,918 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#A#compaction#300 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:12,919 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/415d8e1f02fd4ed5ad65d93fe21882d2 is 175, key is test_row_0/A:col10/1733725029424/Put/seqid=0 2024-12-09T06:17:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T06:17:12,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742176_1352 (size=14794) 2024-12-09T06:17:12,935 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:12,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742177_1353 (size=12459) 2024-12-09T06:17:12,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099ab74533999342a0a905a4c636627b3b_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ab74533999342a0a905a4c636627b3b_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:12,945 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/68dcb971703742c7a3cd55b3f275a25f, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:12,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/68dcb971703742c7a3cd55b3f275a25f is 175, key is test_row_0/A:col10/1733725031681/Put/seqid=0 2024-12-09T06:17:12,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742178_1354 (size=31413) 2024-12-09T06:17:12,961 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/415d8e1f02fd4ed5ad65d93fe21882d2 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/415d8e1f02fd4ed5ad65d93fe21882d2 2024-12-09T06:17:12,968 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/A of 1f2d999c565a1d93300e02727b70ded5 into 415d8e1f02fd4ed5ad65d93fe21882d2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:12,968 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:12,968 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/A, priority=13, startTime=1733725032870; duration=0sec 2024-12-09T06:17:12,968 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:12,968 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:A 2024-12-09T06:17:12,968 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:12,969 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:12,969 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/C is initiating minor compaction (all files) 2024-12-09T06:17:12,969 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/C in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:12,969 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/3f5054c3eee64bc3898ea320f3748968, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fa5e349adc6e4159b92e3ad33a6355d1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/8a67f44eadc243c19208eeefc0d42b8b] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=35.7 K 2024-12-09T06:17:12,970 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5054c3eee64bc3898ea320f3748968, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733725027675 2024-12-09T06:17:12,970 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5e349adc6e4159b92e3ad33a6355d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733725028219 2024-12-09T06:17:12,970 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a67f44eadc243c19208eeefc0d42b8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733725029420 2024-12-09T06:17:12,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725092959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:12,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742179_1355 (size=39749) 2024-12-09T06:17:12,984 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:12,984 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/07a75cfb4c7a41d786e3e8e56194c615 is 50, key is test_row_0/C:col10/1733725029424/Put/seqid=0 2024-12-09T06:17:12,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742180_1356 (size=12459) 2024-12-09T06:17:12,995 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/07a75cfb4c7a41d786e3e8e56194c615 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/07a75cfb4c7a41d786e3e8e56194c615 2024-12-09T06:17:13,002 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/C of 1f2d999c565a1d93300e02727b70ded5 into 07a75cfb4c7a41d786e3e8e56194c615(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:13,002 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:13,002 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/C, priority=13, startTime=1733725032870; duration=0sec 2024-12-09T06:17:13,002 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:13,002 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:C 2024-12-09T06:17:13,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T06:17:13,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:13,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:13,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:13,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725093076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:13,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T06:17:13,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:13,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:13,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:13,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T06:17:13,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725093299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:13,344 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/3fc126140eae47978bbc600a8a16a36e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/3fc126140eae47978bbc600a8a16a36e 2024-12-09T06:17:13,350 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/B of 1f2d999c565a1d93300e02727b70ded5 into 3fc126140eae47978bbc600a8a16a36e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:13,350 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:13,350 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/B, priority=13, startTime=1733725032870; duration=0sec 2024-12-09T06:17:13,350 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:13,350 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:B 2024-12-09T06:17:13,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T06:17:13,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:13,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:13,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:13,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:13,378 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/68dcb971703742c7a3cd55b3f275a25f 2024-12-09T06:17:13,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/26ab8576061f4f8e92642f86cbc38823 is 50, key is test_row_0/B:col10/1733725031681/Put/seqid=0 2024-12-09T06:17:13,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742181_1357 (size=12151) 2024-12-09T06:17:13,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/26ab8576061f4f8e92642f86cbc38823 2024-12-09T06:17:13,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/7e1b03b57a8f4aa49fde1a496007ea31 is 50, key is test_row_0/C:col10/1733725031681/Put/seqid=0 2024-12-09T06:17:13,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742182_1358 (size=12151) 2024-12-09T06:17:13,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/7e1b03b57a8f4aa49fde1a496007ea31 2024-12-09T06:17:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/68dcb971703742c7a3cd55b3f275a25f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/68dcb971703742c7a3cd55b3f275a25f 2024-12-09T06:17:13,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/68dcb971703742c7a3cd55b3f275a25f, entries=200, sequenceid=199, filesize=38.8 K 2024-12-09T06:17:13,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/26ab8576061f4f8e92642f86cbc38823 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/26ab8576061f4f8e92642f86cbc38823 2024-12-09T06:17:13,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/26ab8576061f4f8e92642f86cbc38823, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T06:17:13,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/7e1b03b57a8f4aa49fde1a496007ea31 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/7e1b03b57a8f4aa49fde1a496007ea31 2024-12-09T06:17:13,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/7e1b03b57a8f4aa49fde1a496007ea31, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T06:17:13,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 1f2d999c565a1d93300e02727b70ded5 in 612ms, sequenceid=199, compaction requested=false 2024-12-09T06:17:13,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:13,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T06:17:13,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:13,529 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:17:13,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:13,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:13,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:13,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:13,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:13,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:13,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092b633788ccbb4cdda62f286e337b2d03_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725032957/Put/seqid=0 2024-12-09T06:17:13,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742183_1359 (size=12304) 2024-12-09T06:17:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:13,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T06:17:13,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725033737, max timestamp=1733725033787Num uniques 859; 2024-12-09T06:17:13,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725033737, max timestamp=1733725033787Num uniques 859; 2024-12-09T06:17:13,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725033737, max timestamp=1733725033787Num uniques 859; 2024-12-09T06:17:13,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:13,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:13,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:13,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725093898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:13,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733725033877, max timestamp=1733725033908Num uniques 1018; 2024-12-09T06:17:13,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:17:13,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733725033877, max timestamp=1733725033908Num uniques 1018; 2024-12-09T06:17:13,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:17:13,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733725033877, max timestamp=1733725033908Num uniques 1018; 2024-12-09T06:17:13,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:17:13,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:13,984 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092b633788ccbb4cdda62f286e337b2d03_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092b633788ccbb4cdda62f286e337b2d03_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/9df8b242ae2e4995b2b84342dda3da8a, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/9df8b242ae2e4995b2b84342dda3da8a is 175, key is test_row_0/A:col10/1733725032957/Put/seqid=0 2024-12-09T06:17:13,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742184_1360 (size=31105) 2024-12-09T06:17:13,993 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/9df8b242ae2e4995b2b84342dda3da8a 2024-12-09T06:17:14,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/ee0a28aba63c4e4e95423a724746ffbf is 50, key is test_row_0/B:col10/1733725032957/Put/seqid=0 2024-12-09T06:17:14,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742185_1361 (size=12151) 2024-12-09T06:17:14,023 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/ee0a28aba63c4e4e95423a724746ffbf 2024-12-09T06:17:14,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725094017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/5134342abdcd4519911eb7c69d7de098 is 50, key is test_row_0/C:col10/1733725032957/Put/seqid=0 2024-12-09T06:17:14,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725094020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725094025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742186_1362 (size=12151) 2024-12-09T06:17:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725094147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725094147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725094236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725094356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725094357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,462 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/5134342abdcd4519911eb7c69d7de098 2024-12-09T06:17:14,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/9df8b242ae2e4995b2b84342dda3da8a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/9df8b242ae2e4995b2b84342dda3da8a 2024-12-09T06:17:14,469 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/9df8b242ae2e4995b2b84342dda3da8a, entries=150, sequenceid=212, filesize=30.4 K 2024-12-09T06:17:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/ee0a28aba63c4e4e95423a724746ffbf as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ee0a28aba63c4e4e95423a724746ffbf 2024-12-09T06:17:14,475 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ee0a28aba63c4e4e95423a724746ffbf, entries=150, sequenceid=212, filesize=11.9 K 2024-12-09T06:17:14,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/5134342abdcd4519911eb7c69d7de098 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/5134342abdcd4519911eb7c69d7de098 2024-12-09T06:17:14,480 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/5134342abdcd4519911eb7c69d7de098, entries=150, sequenceid=212, filesize=11.9 K 2024-12-09T06:17:14,481 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1f2d999c565a1d93300e02727b70ded5 in 953ms, sequenceid=212, compaction requested=true 2024-12-09T06:17:14,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:14,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:14,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T06:17:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-09T06:17:14,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T06:17:14,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8760 sec 2024-12-09T06:17:14,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8990 sec 2024-12-09T06:17:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:14,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=140.89 KB heapSize=404.77 KB 2024-12-09T06:17:14,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:14,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:17:14,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:14,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:17:14,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:14,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:17:14,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a8ca86e90eb642bca83272711c781ee2_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725034023/Put/seqid=0 2024-12-09T06:17:14,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742187_1363 (size=14994) 2024-12-09T06:17:14,567 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:14,570 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a8ca86e90eb642bca83272711c781ee2_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a8ca86e90eb642bca83272711c781ee2_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:14,571 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/515569f4d6c94b6fac6d275044056352, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:14,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/515569f4d6c94b6fac6d275044056352 is 175, key is test_row_0/A:col10/1733725034023/Put/seqid=0 2024-12-09T06:17:14,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742188_1364 (size=39949) 2024-12-09T06:17:14,579 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/515569f4d6c94b6fac6d275044056352 2024-12-09T06:17:14,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/d6922ed363434e8198db9f9261a7aac2 is 50, key is test_row_0/B:col10/1733725034023/Put/seqid=0 2024-12-09T06:17:14,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742189_1365 (size=12301) 2024-12-09T06:17:14,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/d6922ed363434e8198db9f9261a7aac2 2024-12-09T06:17:14,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/4b264989fef94531944a071c5e66b3d4 is 50, key is test_row_0/C:col10/1733725034023/Put/seqid=0 2024-12-09T06:17:14,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742190_1366 (size=12301) 2024-12-09T06:17:14,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/4b264989fef94531944a071c5e66b3d4 2024-12-09T06:17:14,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/515569f4d6c94b6fac6d275044056352 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/515569f4d6c94b6fac6d275044056352 2024-12-09T06:17:14,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/515569f4d6c94b6fac6d275044056352, entries=200, sequenceid=267, filesize=39.0 K 2024-12-09T06:17:14,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725094614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/d6922ed363434e8198db9f9261a7aac2 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d6922ed363434e8198db9f9261a7aac2 2024-12-09T06:17:14,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d6922ed363434e8198db9f9261a7aac2, entries=150, sequenceid=267, filesize=12.0 K 2024-12-09T06:17:14,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/4b264989fef94531944a071c5e66b3d4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4b264989fef94531944a071c5e66b3d4 2024-12-09T06:17:14,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4b264989fef94531944a071c5e66b3d4, entries=150, sequenceid=267, filesize=12.0 K 2024-12-09T06:17:14,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~404.72 KB/414432, currentSize=40.25 KB/41220 for 1f2d999c565a1d93300e02727b70ded5 in 77ms, sequenceid=267, compaction requested=true 2024-12-09T06:17:14,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:14,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:14,632 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:14,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:14,632 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:14,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:14,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:14,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:14,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:14,634 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142216 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:14,634 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:14,634 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/A is initiating minor compaction (all files) 2024-12-09T06:17:14,634 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/B is initiating minor compaction (all files) 2024-12-09T06:17:14,634 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/A in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:14,634 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/B in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:14,634 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/415d8e1f02fd4ed5ad65d93fe21882d2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/68dcb971703742c7a3cd55b3f275a25f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/9df8b242ae2e4995b2b84342dda3da8a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/515569f4d6c94b6fac6d275044056352] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=138.9 K 2024-12-09T06:17:14,634 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/3fc126140eae47978bbc600a8a16a36e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/26ab8576061f4f8e92642f86cbc38823, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ee0a28aba63c4e4e95423a724746ffbf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d6922ed363434e8198db9f9261a7aac2] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=47.9 K 2024-12-09T06:17:14,634 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:14,634 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/415d8e1f02fd4ed5ad65d93fe21882d2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/68dcb971703742c7a3cd55b3f275a25f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/9df8b242ae2e4995b2b84342dda3da8a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/515569f4d6c94b6fac6d275044056352] 2024-12-09T06:17:14,634 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fc126140eae47978bbc600a8a16a36e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733725029420 2024-12-09T06:17:14,634 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 415d8e1f02fd4ed5ad65d93fe21882d2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733725029420 2024-12-09T06:17:14,635 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ab8576061f4f8e92642f86cbc38823, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733725031681 2024-12-09T06:17:14,635 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68dcb971703742c7a3cd55b3f275a25f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733725031681 2024-12-09T06:17:14,635 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ee0a28aba63c4e4e95423a724746ffbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733725032920 2024-12-09T06:17:14,635 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9df8b242ae2e4995b2b84342dda3da8a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733725032920 2024-12-09T06:17:14,635 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting d6922ed363434e8198db9f9261a7aac2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733725034008 2024-12-09T06:17:14,635 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 515569f4d6c94b6fac6d275044056352, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733725033991 2024-12-09T06:17:14,641 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:14,642 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:14,642 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/cc86bb3352414a46b1cf137fac93161e is 50, key is test_row_0/B:col10/1733725034023/Put/seqid=0 2024-12-09T06:17:14,642 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120953079281f4d641ebbf64525399110d4c_1f2d999c565a1d93300e02727b70ded5 store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:14,645 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120953079281f4d641ebbf64525399110d4c_1f2d999c565a1d93300e02727b70ded5, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:14,645 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120953079281f4d641ebbf64525399110d4c_1f2d999c565a1d93300e02727b70ded5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:14,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742191_1367 (size=12745) 2024-12-09T06:17:14,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742192_1368 (size=4469) 2024-12-09T06:17:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:14,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:17:14,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:14,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:14,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:14,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:14,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:14,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:14,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090a49201fffd84781a549e7b06cfd5e9b_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725034610/Put/seqid=0 2024-12-09T06:17:14,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742193_1369 (size=14994) 2024-12-09T06:17:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T06:17:14,757 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:14,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-09T06:17:14,759 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:14,760 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:14,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:14,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725094745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725034740, max timestamp=1733725034759Num uniques 859; 2024-12-09T06:17:14,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:14,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725034740, max timestamp=1733725034759Num uniques 859; 2024-12-09T06:17:14,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T06:17:14,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725034740, max timestamp=1733725034759Num uniques 859; 2024-12-09T06:17:14,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:14,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725094812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725094825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725094829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T06:17:14,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725094867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T06:17:14,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:14,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:14,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:14,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:14,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:14,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725094937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725094947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725094947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,051 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:15,052 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/c0d4687c5dc845adb9e31a58234a3253 is 175, key is test_row_0/A:col10/1733725034023/Put/seqid=0 2024-12-09T06:17:15,055 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/cc86bb3352414a46b1cf137fac93161e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/cc86bb3352414a46b1cf137fac93161e 2024-12-09T06:17:15,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742194_1370 (size=31699) 2024-12-09T06:17:15,064 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/B of 1f2d999c565a1d93300e02727b70ded5 into cc86bb3352414a46b1cf137fac93161e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:15,064 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:15,064 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/B, priority=12, startTime=1733725034632; duration=0sec 2024-12-09T06:17:15,065 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:15,065 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:B 2024-12-09T06:17:15,065 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:15,066 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:15,066 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/C is initiating minor compaction (all files) 2024-12-09T06:17:15,066 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/C in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,067 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/07a75cfb4c7a41d786e3e8e56194c615, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/7e1b03b57a8f4aa49fde1a496007ea31, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/5134342abdcd4519911eb7c69d7de098, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4b264989fef94531944a071c5e66b3d4] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=47.9 K 2024-12-09T06:17:15,067 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a75cfb4c7a41d786e3e8e56194c615, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733725029420 2024-12-09T06:17:15,067 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1b03b57a8f4aa49fde1a496007ea31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733725031681 2024-12-09T06:17:15,067 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5134342abdcd4519911eb7c69d7de098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733725032920 2024-12-09T06:17:15,068 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b264989fef94531944a071c5e66b3d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733725034008 2024-12-09T06:17:15,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T06:17:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T06:17:15,081 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:15,081 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/2492137aa4c941b5b2328195f911efea is 50, key is test_row_0/C:col10/1733725034023/Put/seqid=0 2024-12-09T06:17:15,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742195_1371 (size=12745) 2024-12-09T06:17:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725095087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,128 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:15,131 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090a49201fffd84781a549e7b06cfd5e9b_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090a49201fffd84781a549e7b06cfd5e9b_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:15,134 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/b080a52ed1b14e8e81aa2e305214cf98, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:15,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/b080a52ed1b14e8e81aa2e305214cf98 is 175, key is test_row_0/A:col10/1733725034610/Put/seqid=0 2024-12-09T06:17:15,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742196_1372 (size=39949) 2024-12-09T06:17:15,144 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/b080a52ed1b14e8e81aa2e305214cf98 2024-12-09T06:17:15,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725095157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/134aa0edba624032bb7d4c856d5c64bf is 50, key is test_row_0/B:col10/1733725034610/Put/seqid=0 2024-12-09T06:17:15,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725095167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:15,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725095168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742197_1373 (size=12301) 2024-12-09T06:17:15,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/134aa0edba624032bb7d4c856d5c64bf 2024-12-09T06:17:15,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/01e036eccb70409daae2d25b5cff2c5e is 50, key is test_row_0/C:col10/1733725034610/Put/seqid=0 2024-12-09T06:17:15,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742198_1374 (size=12301) 2024-12-09T06:17:15,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T06:17:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T06:17:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T06:17:15,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725095409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,463 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/c0d4687c5dc845adb9e31a58234a3253 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0d4687c5dc845adb9e31a58234a3253 2024-12-09T06:17:15,472 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/A of 1f2d999c565a1d93300e02727b70ded5 into c0d4687c5dc845adb9e31a58234a3253(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:15,472 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:15,472 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/A, priority=12, startTime=1733725034631; duration=0sec 2024-12-09T06:17:15,472 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:15,472 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:A 2024-12-09T06:17:15,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725095476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,493 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/2492137aa4c941b5b2328195f911efea as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2492137aa4c941b5b2328195f911efea 2024-12-09T06:17:15,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725095487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:15,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:15,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725095487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:15,506 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/C of 1f2d999c565a1d93300e02727b70ded5 into 2492137aa4c941b5b2328195f911efea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:15,507 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:15,507 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/C, priority=12, startTime=1733725034632; duration=0sec 2024-12-09T06:17:15,507 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:15,507 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:C 2024-12-09T06:17:15,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T06:17:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:15,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/01e036eccb70409daae2d25b5cff2c5e 2024-12-09T06:17:15,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/b080a52ed1b14e8e81aa2e305214cf98 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/b080a52ed1b14e8e81aa2e305214cf98 2024-12-09T06:17:15,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/b080a52ed1b14e8e81aa2e305214cf98, entries=200, sequenceid=278, filesize=39.0 K 2024-12-09T06:17:15,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/134aa0edba624032bb7d4c856d5c64bf as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/134aa0edba624032bb7d4c856d5c64bf 2024-12-09T06:17:15,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/134aa0edba624032bb7d4c856d5c64bf, entries=150, sequenceid=278, filesize=12.0 K 2024-12-09T06:17:15,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/01e036eccb70409daae2d25b5cff2c5e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/01e036eccb70409daae2d25b5cff2c5e 2024-12-09T06:17:15,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/01e036eccb70409daae2d25b5cff2c5e, entries=150, sequenceid=278, filesize=12.0 K 2024-12-09T06:17:15,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1f2d999c565a1d93300e02727b70ded5 in 966ms, sequenceid=278, compaction requested=false 2024-12-09T06:17:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:15,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T06:17:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:15,694 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T06:17:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120997be06ccf64547cb97906cbc73cb4fcb_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725034824/Put/seqid=0 2024-12-09T06:17:15,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742199_1375 (size=12454) 2024-12-09T06:17:15,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:15,800 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120997be06ccf64547cb97906cbc73cb4fcb_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120997be06ccf64547cb97906cbc73cb4fcb_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:15,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/5a9d9c542edd42c2bbbbaab83a892348, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:15,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/5a9d9c542edd42c2bbbbaab83a892348 is 175, key is test_row_0/A:col10/1733725034824/Put/seqid=0 2024-12-09T06:17:15,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742200_1376 (size=31255) 2024-12-09T06:17:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T06:17:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:15,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725095999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725096007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725096007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725096008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725096117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725096126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,237 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/5a9d9c542edd42c2bbbbaab83a892348 2024-12-09T06:17:16,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/4c107f11ea824432bfc23a60293ac4ce is 50, key is test_row_0/B:col10/1733725034824/Put/seqid=0 2024-12-09T06:17:16,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725096336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725096347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59680 deadline: 1733725096514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,521 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 1f2d999c565a1d93300e02727b70ded5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8275 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742201_1377 (size=12301) 2024-12-09T06:17:16,630 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/4c107f11ea824432bfc23a60293ac4ce 2024-12-09T06:17:16,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/6b9dd0be1464416cbb46469eb02fd947 is 50, key is test_row_0/C:col10/1733725034824/Put/seqid=0 2024-12-09T06:17:16,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725096647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725096666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:16,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742202_1378 (size=12301) 2024-12-09T06:17:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T06:17:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725097027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725097027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,034 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 1f2d999c565a1d93300e02727b70ded5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,036 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in 1f2d999c565a1d93300e02727b70ded5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,078 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/6b9dd0be1464416cbb46469eb02fd947 2024-12-09T06:17:17,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/5a9d9c542edd42c2bbbbaab83a892348 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5a9d9c542edd42c2bbbbaab83a892348 2024-12-09T06:17:17,093 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5a9d9c542edd42c2bbbbaab83a892348, entries=150, sequenceid=316, filesize=30.5 K 2024-12-09T06:17:17,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/4c107f11ea824432bfc23a60293ac4ce as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4c107f11ea824432bfc23a60293ac4ce 2024-12-09T06:17:17,097 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4c107f11ea824432bfc23a60293ac4ce, entries=150, sequenceid=316, filesize=12.0 K 2024-12-09T06:17:17,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/6b9dd0be1464416cbb46469eb02fd947 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/6b9dd0be1464416cbb46469eb02fd947 2024-12-09T06:17:17,102 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/6b9dd0be1464416cbb46469eb02fd947, entries=150, sequenceid=316, filesize=12.0 K 2024-12-09T06:17:17,103 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 1f2d999c565a1d93300e02727b70ded5 in 1409ms, sequenceid=316, compaction requested=true 2024-12-09T06:17:17,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:17,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:17,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-09T06:17:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-09T06:17:17,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-09T06:17:17,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3440 sec 2024-12-09T06:17:17,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3510 sec 2024-12-09T06:17:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:17,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T06:17:17,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:17,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:17,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:17,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:17,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:17,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:17,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e34e7e2a1f8144bf806056cf4ad1afeb_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725035997/Put/seqid=0 2024-12-09T06:17:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742203_1379 (size=14994) 2024-12-09T06:17:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725097256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725097257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725097381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725097381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:17,606 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e34e7e2a1f8144bf806056cf4ad1afeb_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e34e7e2a1f8144bf806056cf4ad1afeb_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:17,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/3ee947ef2f0b42b48362496ce960531a, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/3ee947ef2f0b42b48362496ce960531a is 175, key is test_row_0/A:col10/1733725035997/Put/seqid=0 2024-12-09T06:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725097598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725097598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742204_1380 (size=39949) 2024-12-09T06:17:17,648 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/3ee947ef2f0b42b48362496ce960531a 2024-12-09T06:17:17,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/4a119881a111466f9f51aad32dd55df7 is 50, key is test_row_0/B:col10/1733725035997/Put/seqid=0 2024-12-09T06:17:17,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742205_1381 (size=12301) 2024-12-09T06:17:17,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725097917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725097917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:18,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/4a119881a111466f9f51aad32dd55df7 2024-12-09T06:17:18,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/c862a414584741c494303bf963a69847 is 50, key is test_row_0/C:col10/1733725035997/Put/seqid=0 2024-12-09T06:17:18,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742206_1382 (size=12301) 2024-12-09T06:17:18,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725098437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:18,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725098437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:18,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/c862a414584741c494303bf963a69847 2024-12-09T06:17:18,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/3ee947ef2f0b42b48362496ce960531a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/3ee947ef2f0b42b48362496ce960531a 2024-12-09T06:17:18,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/3ee947ef2f0b42b48362496ce960531a, entries=200, sequenceid=328, filesize=39.0 K 2024-12-09T06:17:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/4a119881a111466f9f51aad32dd55df7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4a119881a111466f9f51aad32dd55df7 2024-12-09T06:17:18,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4a119881a111466f9f51aad32dd55df7, entries=150, sequenceid=328, filesize=12.0 K 2024-12-09T06:17:18,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/c862a414584741c494303bf963a69847 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/c862a414584741c494303bf963a69847 2024-12-09T06:17:18,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/c862a414584741c494303bf963a69847, entries=150, sequenceid=328, filesize=12.0 K 2024-12-09T06:17:18,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1f2d999c565a1d93300e02727b70ded5 in 1370ms, sequenceid=328, compaction requested=true 2024-12-09T06:17:18,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:18,541 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:18,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:18,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:18,541 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:18,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:18,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:18,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:18,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:18,542 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:18,542 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/B is initiating minor compaction (all files) 2024-12-09T06:17:18,542 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/B in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:18,542 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/cc86bb3352414a46b1cf137fac93161e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/134aa0edba624032bb7d4c856d5c64bf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4c107f11ea824432bfc23a60293ac4ce, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4a119881a111466f9f51aad32dd55df7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=48.5 K 2024-12-09T06:17:18,543 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:18,543 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/A is initiating minor compaction (all files) 2024-12-09T06:17:18,543 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/A in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:18,543 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting cc86bb3352414a46b1cf137fac93161e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733725034008 2024-12-09T06:17:18,543 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0d4687c5dc845adb9e31a58234a3253, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/b080a52ed1b14e8e81aa2e305214cf98, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5a9d9c542edd42c2bbbbaab83a892348, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/3ee947ef2f0b42b48362496ce960531a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=139.5 K 2024-12-09T06:17:18,543 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:18,543 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0d4687c5dc845adb9e31a58234a3253, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/b080a52ed1b14e8e81aa2e305214cf98, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5a9d9c542edd42c2bbbbaab83a892348, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/3ee947ef2f0b42b48362496ce960531a] 2024-12-09T06:17:18,543 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 134aa0edba624032bb7d4c856d5c64bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733725034610 2024-12-09T06:17:18,544 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d4687c5dc845adb9e31a58234a3253, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733725034008 2024-12-09T06:17:18,544 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c107f11ea824432bfc23a60293ac4ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733725034810 2024-12-09T06:17:18,544 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting b080a52ed1b14e8e81aa2e305214cf98, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733725034610 2024-12-09T06:17:18,544 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a119881a111466f9f51aad32dd55df7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733725035997 2024-12-09T06:17:18,545 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a9d9c542edd42c2bbbbaab83a892348, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733725034810 2024-12-09T06:17:18,545 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ee947ef2f0b42b48362496ce960531a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733725035960 2024-12-09T06:17:18,565 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:18,567 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:18,568 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/a31068bdd56b47f9a3a9dace42123d6d is 50, key is test_row_0/B:col10/1733725035997/Put/seqid=0 2024-12-09T06:17:18,581 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209b0901eee0fd145c6b13382682f270464_1f2d999c565a1d93300e02727b70ded5 store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:18,584 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209b0901eee0fd145c6b13382682f270464_1f2d999c565a1d93300e02727b70ded5, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:18,584 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b0901eee0fd145c6b13382682f270464_1f2d999c565a1d93300e02727b70ded5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:18,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742207_1383 (size=12881) 2024-12-09T06:17:18,622 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/a31068bdd56b47f9a3a9dace42123d6d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a31068bdd56b47f9a3a9dace42123d6d 2024-12-09T06:17:18,628 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/B of 1f2d999c565a1d93300e02727b70ded5 into a31068bdd56b47f9a3a9dace42123d6d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:18,628 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:18,628 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/B, priority=12, startTime=1733725038541; duration=0sec 2024-12-09T06:17:18,628 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:18,628 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:B 2024-12-09T06:17:18,628 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:18,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742208_1384 (size=4469) 2024-12-09T06:17:18,636 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#A#compaction#325 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:18,637 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/338be9d6e9244ca4831c008330df4a6b is 175, key is test_row_0/A:col10/1733725035997/Put/seqid=0 2024-12-09T06:17:18,642 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:18,642 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/C is initiating minor compaction (all files) 2024-12-09T06:17:18,642 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/C in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:18,643 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2492137aa4c941b5b2328195f911efea, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/01e036eccb70409daae2d25b5cff2c5e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/6b9dd0be1464416cbb46469eb02fd947, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/c862a414584741c494303bf963a69847] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=48.5 K 2024-12-09T06:17:18,643 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2492137aa4c941b5b2328195f911efea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733725034008 2024-12-09T06:17:18,643 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e036eccb70409daae2d25b5cff2c5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733725034610 2024-12-09T06:17:18,644 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b9dd0be1464416cbb46469eb02fd947, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733725034810 2024-12-09T06:17:18,644 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c862a414584741c494303bf963a69847, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733725035997 2024-12-09T06:17:18,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742209_1385 (size=31835) 2024-12-09T06:17:18,668 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#C#compaction#326 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:18,669 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/2966ddb6b9a242dd9893f29268dda9a0 is 50, key is test_row_0/C:col10/1733725035997/Put/seqid=0 2024-12-09T06:17:18,670 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/338be9d6e9244ca4831c008330df4a6b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/338be9d6e9244ca4831c008330df4a6b 2024-12-09T06:17:18,675 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/A of 1f2d999c565a1d93300e02727b70ded5 into 338be9d6e9244ca4831c008330df4a6b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:18,675 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:18,675 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/A, priority=12, startTime=1733725038541; duration=0sec 2024-12-09T06:17:18,675 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:18,675 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:A 2024-12-09T06:17:18,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742210_1386 (size=12881) 2024-12-09T06:17:18,706 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/2966ddb6b9a242dd9893f29268dda9a0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2966ddb6b9a242dd9893f29268dda9a0 2024-12-09T06:17:18,716 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/C of 1f2d999c565a1d93300e02727b70ded5 into 2966ddb6b9a242dd9893f29268dda9a0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:18,716 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:18,716 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/C, priority=12, startTime=1733725038541; duration=0sec 2024-12-09T06:17:18,716 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:18,716 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:C 2024-12-09T06:17:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T06:17:18,927 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:18,928 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-09T06:17:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T06:17:18,933 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:18,934 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:18,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T06:17:19,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725037239, max timestamp=1733725039047Num uniques 859; 2024-12-09T06:17:19,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:19,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-09T06:17:19,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725037239, max timestamp=1733725039047Num uniques 859; 2024-12-09T06:17:19,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:19,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725037239, max timestamp=1733725039047Num uniques 859; 2024-12-09T06:17:19,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:19,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093b20248d7be24ec884227f450705ba00_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725039048/Put/seqid=0 2024-12-09T06:17:19,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:19,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:19,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742211_1387 (size=17534) 2024-12-09T06:17:19,135 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:19,141 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093b20248d7be24ec884227f450705ba00_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093b20248d7be24ec884227f450705ba00_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:19,143 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/4498451d0b894efa9b84d1c27e22a7b6, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:19,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/4498451d0b894efa9b84d1c27e22a7b6 is 175, key is test_row_0/A:col10/1733725039048/Put/seqid=0 2024-12-09T06:17:19,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725039122, max timestamp=1733725039156Num uniques 859; 2024-12-09T06:17:19,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:19,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725039122, max timestamp=1733725039156Num uniques 859; 2024-12-09T06:17:19,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:19,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725039122, max timestamp=1733725039156Num uniques 859; 2024-12-09T06:17:19,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:19,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742212_1388 (size=48639) 2024-12-09T06:17:19,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/4498451d0b894efa9b84d1c27e22a7b6 2024-12-09T06:17:19,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/a09075fcc2f440658c56c1642db0304d is 50, key is test_row_0/B:col10/1733725039048/Put/seqid=0 2024-12-09T06:17:19,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742213_1389 (size=12301) 2024-12-09T06:17:19,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/a09075fcc2f440658c56c1642db0304d 2024-12-09T06:17:19,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/f9b1d35a9bf9423590124bc9a5e9c3a7 is 50, key is test_row_0/C:col10/1733725039048/Put/seqid=0 2024-12-09T06:17:19,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742214_1390 (size=12301) 2024-12-09T06:17:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T06:17:19,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:19,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:19,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725039198, max timestamp=1733725039269Num uniques 1018; 2024-12-09T06:17:19,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:17:19,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725039198, max timestamp=1733725039269Num uniques 1018; 2024-12-09T06:17:19,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:17:19,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725039198, max timestamp=1733725039269Num uniques 1018; 2024-12-09T06:17:19,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:17:19,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733725039322, max timestamp=1733725039350Num uniques 1077; 2024-12-09T06:17:19,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733725039322, max timestamp=1733725039350Num uniques 1077; 2024-12-09T06:17:19,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-09T06:17:19,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-09T06:17:19,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733725039322, max timestamp=1733725039350Num uniques 1077; 2024-12-09T06:17:19,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-09T06:17:19,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733725039377, max timestamp=1733725039408Num uniques 1036; 2024-12-09T06:17:19,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733725039377, max timestamp=1733725039408Num uniques 1036; 2024-12-09T06:17:19,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-09T06:17:19,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-09T06:17:19,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733725039377, max timestamp=1733725039408Num uniques 1036; 2024-12-09T06:17:19,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-09T06:17:19,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733725039430, max timestamp=1733725039463Num uniques 1095; 2024-12-09T06:17:19,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733725039430, max timestamp=1733725039463Num uniques 1095; 2024-12-09T06:17:19,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733725039430, max timestamp=1733725039463Num uniques 1095; 2024-12-09T06:17:19,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-09T06:17:19,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-09T06:17:19,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-09T06:17:19,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725099494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733725039473, max timestamp=1733725039496Num uniques 1104; 2024-12-09T06:17:19,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-12-09T06:17:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733725039473, max timestamp=1733725039496Num uniques 1104; 2024-12-09T06:17:19,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-12-09T06:17:19,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733725039473, max timestamp=1733725039496Num uniques 1104; 2024-12-09T06:17:19,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-12-09T06:17:19,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:19,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:19,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725099549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T06:17:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725099552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725099553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725099606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/f9b1d35a9bf9423590124bc9a5e9c3a7 2024-12-09T06:17:19,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/4498451d0b894efa9b84d1c27e22a7b6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/4498451d0b894efa9b84d1c27e22a7b6 2024-12-09T06:17:19,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/4498451d0b894efa9b84d1c27e22a7b6, entries=250, sequenceid=353, filesize=47.5 K 2024-12-09T06:17:19,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/a09075fcc2f440658c56c1642db0304d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a09075fcc2f440658c56c1642db0304d 2024-12-09T06:17:19,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a09075fcc2f440658c56c1642db0304d, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T06:17:19,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/f9b1d35a9bf9423590124bc9a5e9c3a7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/f9b1d35a9bf9423590124bc9a5e9c3a7 2024-12-09T06:17:19,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/f9b1d35a9bf9423590124bc9a5e9c3a7, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T06:17:19,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 1f2d999c565a1d93300e02727b70ded5 in 610ms, sequenceid=353, compaction requested=false 2024-12-09T06:17:19,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:19,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=161.02 KB heapSize=451.78 KB 2024-12-09T06:17:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:17:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:17:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:17:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:19,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c92653a4a3cd427c99568e488b597a46_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725039531/Put/seqid=0 2024-12-09T06:17:19,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725099679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725099680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725099689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:19,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742215_1391 (size=14994) 2024-12-09T06:17:19,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,708 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c92653a4a3cd427c99568e488b597a46_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c92653a4a3cd427c99568e488b597a46_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:19,716 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/280c6f539c0141a19db78c279a8f5e08, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:19,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/280c6f539c0141a19db78c279a8f5e08 is 175, key is test_row_0/A:col10/1733725039531/Put/seqid=0 2024-12-09T06:17:19,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742216_1392 (size=39949) 2024-12-09T06:17:19,739 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=477, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/280c6f539c0141a19db78c279a8f5e08 2024-12-09T06:17:19,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/d505eeefea6d473aa4c20718a3fccc2e is 50, key is test_row_0/B:col10/1733725039531/Put/seqid=0 2024-12-09T06:17:19,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742217_1393 (size=12301) 2024-12-09T06:17:19,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/d505eeefea6d473aa4c20718a3fccc2e 2024-12-09T06:17:19,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/0c9700b830f64ce4a6c277e570e98a7d is 50, key is test_row_0/C:col10/1733725039531/Put/seqid=0 2024-12-09T06:17:19,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742218_1394 (size=12301) 2024-12-09T06:17:19,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725099796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725099796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725099796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725099827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:19,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:19,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:19,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:19,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:19,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:20,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725100006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:20,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:20,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:20,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:20,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:20,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725100017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:20,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725100018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T06:17:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725100146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:20,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:20,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/0c9700b830f64ce4a6c277e570e98a7d 2024-12-09T06:17:20,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/280c6f539c0141a19db78c279a8f5e08 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/280c6f539c0141a19db78c279a8f5e08 2024-12-09T06:17:20,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/280c6f539c0141a19db78c279a8f5e08, entries=200, sequenceid=477, filesize=39.0 K 2024-12-09T06:17:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/d505eeefea6d473aa4c20718a3fccc2e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d505eeefea6d473aa4c20718a3fccc2e 2024-12-09T06:17:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d505eeefea6d473aa4c20718a3fccc2e, entries=150, sequenceid=477, filesize=12.0 K 2024-12-09T06:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/0c9700b830f64ce4a6c277e570e98a7d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0c9700b830f64ce4a6c277e570e98a7d 2024-12-09T06:17:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0c9700b830f64ce4a6c277e570e98a7d, entries=150, sequenceid=477, filesize=12.0 K 2024-12-09T06:17:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.73 KB/462576, currentSize=20.13 KB/20610 for 1f2d999c565a1d93300e02727b70ded5 in 565ms, sequenceid=477, compaction requested=true 2024-12-09T06:17:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:20,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:20,238 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:20,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:20,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:20,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:20,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:17:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,239 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,239 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120423 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,239 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/A is initiating minor compaction (all files) 2024-12-09T06:17:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,240 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/A in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,240 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/338be9d6e9244ca4831c008330df4a6b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/4498451d0b894efa9b84d1c27e22a7b6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/280c6f539c0141a19db78c279a8f5e08] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=117.6 K 2024-12-09T06:17:20,240 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,240 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/338be9d6e9244ca4831c008330df4a6b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/4498451d0b894efa9b84d1c27e22a7b6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/280c6f539c0141a19db78c279a8f5e08] 2024-12-09T06:17:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,240 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:20,240 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/B is initiating minor compaction (all files) 2024-12-09T06:17:20,240 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/B in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,240 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 338be9d6e9244ca4831c008330df4a6b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733725035997 2024-12-09T06:17:20,240 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a31068bdd56b47f9a3a9dace42123d6d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a09075fcc2f440658c56c1642db0304d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d505eeefea6d473aa4c20718a3fccc2e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=36.6 K 2024-12-09T06:17:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,241 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4498451d0b894efa9b84d1c27e22a7b6, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733725037239 2024-12-09T06:17:20,241 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a31068bdd56b47f9a3a9dace42123d6d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733725035997 2024-12-09T06:17:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,241 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 280c6f539c0141a19db78c279a8f5e08, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733725039531 2024-12-09T06:17:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,241 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a09075fcc2f440658c56c1642db0304d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733725037239 2024-12-09T06:17:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,242 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting d505eeefea6d473aa4c20718a3fccc2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733725039531 2024-12-09T06:17:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,248 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,249 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,250 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/470660e3139f491fbe01f30c588d8231 is 50, key is test_row_0/B:col10/1733725039531/Put/seqid=0 2024-12-09T06:17:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,251 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209f02b07870a8247a0b060ec571205030c_1f2d999c565a1d93300e02727b70ded5 store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:20,253 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209f02b07870a8247a0b060ec571205030c_1f2d999c565a1d93300e02727b70ded5, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:20,253 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f02b07870a8247a0b060ec571205030c_1f2d999c565a1d93300e02727b70ded5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742219_1395 (size=12983) 2024-12-09T06:17:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742220_1396 (size=4469) 2024-12-09T06:17:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,276 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/470660e3139f491fbe01f30c588d8231 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/470660e3139f491fbe01f30c588d8231 2024-12-09T06:17:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,277 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#A#compaction#334 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,277 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/26598bce8533482baaae8a17f0850ee3 is 175, key is test_row_0/A:col10/1733725039531/Put/seqid=0 2024-12-09T06:17:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,283 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/B of 1f2d999c565a1d93300e02727b70ded5 into 470660e3139f491fbe01f30c588d8231(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,283 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,283 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/B, priority=13, startTime=1733725040238; duration=0sec 2024-12-09T06:17:20,283 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:20,283 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:B 2024-12-09T06:17:20,283 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,287 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,287 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/C is initiating minor compaction (all files) 2024-12-09T06:17:20,287 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/C in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,287 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2966ddb6b9a242dd9893f29268dda9a0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/f9b1d35a9bf9423590124bc9a5e9c3a7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0c9700b830f64ce4a6c277e570e98a7d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=36.6 K 2024-12-09T06:17:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,287 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2966ddb6b9a242dd9893f29268dda9a0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733725035997 2024-12-09T06:17:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,288 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b1d35a9bf9423590124bc9a5e9c3a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733725037239 2024-12-09T06:17:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,288 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9700b830f64ce4a6c277e570e98a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733725039531 2024-12-09T06:17:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,296 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,296 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/eb1dbba70a1f4827b71f09319d7240ad is 50, key is test_row_0/C:col10/1733725039531/Put/seqid=0 2024-12-09T06:17:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742221_1397 (size=31937) 2024-12-09T06:17:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,303 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/26598bce8533482baaae8a17f0850ee3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/26598bce8533482baaae8a17f0850ee3 2024-12-09T06:17:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742222_1398 (size=12983) 2024-12-09T06:17:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,309 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/A of 1f2d999c565a1d93300e02727b70ded5 into 26598bce8533482baaae8a17f0850ee3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,309 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:20,309 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/A, priority=13, startTime=1733725040238; duration=0sec 2024-12-09T06:17:20,310 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,310 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:A 2024-12-09T06:17:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,312 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/eb1dbba70a1f4827b71f09319d7240ad as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/eb1dbba70a1f4827b71f09319d7240ad 2024-12-09T06:17:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,317 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/C of 1f2d999c565a1d93300e02727b70ded5 into eb1dbba70a1f4827b71f09319d7240ad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:20,317 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,317 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/C, priority=13, startTime=1733725040238; duration=0sec 2024-12-09T06:17:20,318 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:20,318 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:C 2024-12-09T06:17:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T06:17:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,321 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-09T06:17:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ffcf6036a5384d84b80e23742e95f0cf_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725039688/Put/seqid=0 2024-12-09T06:17:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742223_1399 (size=9914) 2024-12-09T06:17:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:20,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725040545, max timestamp=1733725040593Num uniques 859; 2024-12-09T06:17:20,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:20,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725040545, max timestamp=1733725040593Num uniques 859; 2024-12-09T06:17:20,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:20,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725040545, max timestamp=1733725040593Num uniques 859; 2024-12-09T06:17:20,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:20,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725040633, max timestamp=1733725040650Num uniques 1068; 2024-12-09T06:17:20,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T06:17:20,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725040633, max timestamp=1733725040650Num uniques 1068; 2024-12-09T06:17:20,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T06:17:20,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725040633, max timestamp=1733725040650Num uniques 1068; 2024-12-09T06:17:20,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T06:17:20,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733725040684, max timestamp=1733725040693Num uniques 1077; 2024-12-09T06:17:20,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T06:17:20,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733725040684, max timestamp=1733725040693Num uniques 1077; 2024-12-09T06:17:20,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T06:17:20,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733725040684, max timestamp=1733725040693Num uniques 1077; 2024-12-09T06:17:20,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-09T06:17:20,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733725040714, max timestamp=1733725040730Num uniques 1086; 2024-12-09T06:17:20,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-12-09T06:17:20,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733725040714, max timestamp=1733725040730Num uniques 1086; 2024-12-09T06:17:20,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733725040714, max timestamp=1733725040730Num uniques 1086; 2024-12-09T06:17:20,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-12-09T06:17:20,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-12-09T06:17:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:20,772 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ffcf6036a5384d84b80e23742e95f0cf_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ffcf6036a5384d84b80e23742e95f0cf_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:20,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/371af65bb2694edc8a971bd52c7e8dce, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/371af65bb2694edc8a971bd52c7e8dce is 175, key is test_row_0/A:col10/1733725039688/Put/seqid=0 2024-12-09T06:17:20,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733725040755, max timestamp=1733725040774Num uniques 1195; 2024-12-09T06:17:20,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733725040755, max timestamp=1733725040774Num uniques 1195; 2024-12-09T06:17:20,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-12-09T06:17:20,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-12-09T06:17:20,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733725040755, max timestamp=1733725040774Num uniques 1195; 2024-12-09T06:17:20,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-12-09T06:17:20,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742224_1400 (size=22561) 2024-12-09T06:17:20,784 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=486, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/371af65bb2694edc8a971bd52c7e8dce 2024-12-09T06:17:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/e1049b0208c4401bba15a1d402acd352 is 50, key is test_row_0/B:col10/1733725039688/Put/seqid=0 2024-12-09T06:17:20,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742225_1401 (size=9857) 2024-12-09T06:17:20,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733725040811, max timestamp=1733725040816Num uniques 1104; 2024-12-09T06:17:20,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733725040811, max timestamp=1733725040816Num uniques 1104; 2024-12-09T06:17:20,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733725040811, max timestamp=1733725040816Num uniques 1104; 2024-12-09T06:17:20,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-09T06:17:20,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-09T06:17:20,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-09T06:17:20,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733725040833, max timestamp=1733725040888Num uniques 1013; 2024-12-09T06:17:20,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733725040833, max timestamp=1733725040888Num uniques 1013; 2024-12-09T06:17:20,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-09T06:17:20,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-09T06:17:20,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733725040833, max timestamp=1733725040888Num uniques 1013; 2024-12-09T06:17:20,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-09T06:17:20,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733725040911, max timestamp=1733725040939Num uniques 1022; 2024-12-09T06:17:20,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-09T06:17:20,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733725040911, max timestamp=1733725040939Num uniques 1022; 2024-12-09T06:17:20,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-09T06:17:20,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733725040911, max timestamp=1733725040939Num uniques 1022; 2024-12-09T06:17:20,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-09T06:17:20,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12858, totalHeapSize=29464, min timestamp=1733725040952, max timestamp=1733725040994Num uniques 1081; 2024-12-09T06:17:20,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12856, totalHeapSize=29464, min timestamp=1733725040952, max timestamp=1733725040994Num uniques 1081; 2024-12-09T06:17:20,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12856, totalHeapSize=29464, min timestamp=1733725040952, max timestamp=1733725040994Num uniques 1081; 2024-12-09T06:17:20,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49498, new segment data size=12858, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-12-09T06:17:20,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=12856, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-12-09T06:17:20,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=12856, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-12-09T06:17:21,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733725041033, max timestamp=1733725041050Num uniques 1140; 2024-12-09T06:17:21,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-12-09T06:17:21,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733725041033, max timestamp=1733725041050Num uniques 1140; 2024-12-09T06:17:21,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-12-09T06:17:21,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733725041033, max timestamp=1733725041050Num uniques 1140; 2024-12-09T06:17:21,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-12-09T06:17:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T06:17:21,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1733725041077, max timestamp=1733725041093Num uniques 1099; 2024-12-09T06:17:21,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1733725041077, max timestamp=1733725041093Num uniques 1099; 2024-12-09T06:17:21,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-12-09T06:17:21,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-12-09T06:17:21,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1733725041077, max timestamp=1733725041093Num uniques 1099; 2024-12-09T06:17:21,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-12-09T06:17:21,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1733725041106, max timestamp=1733725041173Num uniques 1208; 2024-12-09T06:17:21,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=55318, new segment data size=9518, suffix heap size=153600,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=208 2024-12-09T06:17:21,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1733725041106, max timestamp=1733725041173Num uniques 1208; 2024-12-09T06:17:21,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55318, new segment data size=9518, suffix heap size=153600,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=208 2024-12-09T06:17:21,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1733725041106, max timestamp=1733725041173Num uniques 1208; 2024-12-09T06:17:21,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=55318, new segment data size=9518, suffix heap size=153600,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=208 2024-12-09T06:17:21,204 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/e1049b0208c4401bba15a1d402acd352 2024-12-09T06:17:21,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/4f12b54a46f042f198ec5c54c1717860 is 50, key is test_row_0/C:col10/1733725039688/Put/seqid=0 2024-12-09T06:17:21,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742226_1402 (size=9857) 2024-12-09T06:17:21,245 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/4f12b54a46f042f198ec5c54c1717860 2024-12-09T06:17:21,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/371af65bb2694edc8a971bd52c7e8dce as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/371af65bb2694edc8a971bd52c7e8dce 2024-12-09T06:17:21,273 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/371af65bb2694edc8a971bd52c7e8dce, entries=100, sequenceid=486, filesize=22.0 K 2024-12-09T06:17:21,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725101262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733725041232, max timestamp=1733725041270Num uniques 1067; 2024-12-09T06:17:21,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-09T06:17:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733725041232, max timestamp=1733725041270Num uniques 1067; 2024-12-09T06:17:21,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-09T06:17:21,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/e1049b0208c4401bba15a1d402acd352 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/e1049b0208c4401bba15a1d402acd352 2024-12-09T06:17:21,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733725041232, max timestamp=1733725041270Num uniques 1067; 2024-12-09T06:17:21,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-09T06:17:21,291 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/e1049b0208c4401bba15a1d402acd352, entries=100, sequenceid=486, filesize=9.6 K 2024-12-09T06:17:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/4f12b54a46f042f198ec5c54c1717860 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4f12b54a46f042f198ec5c54c1717860 2024-12-09T06:17:21,299 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4f12b54a46f042f198ec5c54c1717860, entries=100, sequenceid=486, filesize=9.6 K 2024-12-09T06:17:21,305 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=67.09 KB/68700 for 1f2d999c565a1d93300e02727b70ded5 in 984ms, sequenceid=486, compaction requested=false 2024-12-09T06:17:21,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:21,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:21,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-09T06:17:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:21,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=67.09 KB heapSize=185.02 KB 2024-12-09T06:17:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-09T06:17:21,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-09T06:17:21,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3720 sec 2024-12-09T06:17:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:21,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:21,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:21,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3800 sec 2024-12-09T06:17:21,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209624547aeeb1144ec8ffd30d0bc35a289_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725041303/Put/seqid=0 2024-12-09T06:17:21,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742227_1403 (size=14994) 2024-12-09T06:17:21,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:21,367 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209624547aeeb1144ec8ffd30d0bc35a289_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209624547aeeb1144ec8ffd30d0bc35a289_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:21,370 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/2664eb200a9c40ec9d188e070af90f51, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:21,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/2664eb200a9c40ec9d188e070af90f51 is 175, key is test_row_0/A:col10/1733725041303/Put/seqid=0 2024-12-09T06:17:21,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742228_1404 (size=39949) 2024-12-09T06:17:21,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725101393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725101403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725101405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725101406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725101516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725101527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725101527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725101527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725101737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725101737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725101737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725101741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:21,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=720, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/2664eb200a9c40ec9d188e070af90f51 2024-12-09T06:17:21,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/69303300a49a4e97912454032b088702 is 50, key is test_row_0/B:col10/1733725041303/Put/seqid=0 2024-12-09T06:17:21,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742229_1405 (size=12301) 2024-12-09T06:17:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725102049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725102056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725102056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:22,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725102066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:22,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/69303300a49a4e97912454032b088702 2024-12-09T06:17:22,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/802d1ffb3c1b4033a8d6e9f83ed14b90 is 50, key is test_row_0/C:col10/1733725041303/Put/seqid=0 2024-12-09T06:17:22,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742230_1406 (size=12301) 2024-12-09T06:17:22,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725102573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:22,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725102573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:22,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725102580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:22,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725102588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:22,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:22,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/802d1ffb3c1b4033a8d6e9f83ed14b90 2024-12-09T06:17:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/2664eb200a9c40ec9d188e070af90f51 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/2664eb200a9c40ec9d188e070af90f51 2024-12-09T06:17:22,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/2664eb200a9c40ec9d188e070af90f51, entries=200, sequenceid=720, filesize=39.0 K 2024-12-09T06:17:22,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/69303300a49a4e97912454032b088702 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/69303300a49a4e97912454032b088702 2024-12-09T06:17:22,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/69303300a49a4e97912454032b088702, entries=150, sequenceid=720, filesize=12.0 K 2024-12-09T06:17:22,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/802d1ffb3c1b4033a8d6e9f83ed14b90 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/802d1ffb3c1b4033a8d6e9f83ed14b90 2024-12-09T06:17:22,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/802d1ffb3c1b4033a8d6e9f83ed14b90, entries=150, sequenceid=720, filesize=12.0 K 2024-12-09T06:17:22,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~204.89 KB/209808, currentSize=107.34 KB/109920 for 1f2d999c565a1d93300e02727b70ded5 in 1409ms, sequenceid=720, compaction requested=true 2024-12-09T06:17:22,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:22,715 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:22,715 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:22,719 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:22,719 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/B is initiating minor compaction (all files) 2024-12-09T06:17:22,719 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/B in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:22,719 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/470660e3139f491fbe01f30c588d8231, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/e1049b0208c4401bba15a1d402acd352, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/69303300a49a4e97912454032b088702] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=34.3 K 2024-12-09T06:17:22,720 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:22,720 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/A is initiating minor compaction (all files) 2024-12-09T06:17:22,720 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/A in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:22,720 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/26598bce8533482baaae8a17f0850ee3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/371af65bb2694edc8a971bd52c7e8dce, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/2664eb200a9c40ec9d188e070af90f51] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=92.2 K 2024-12-09T06:17:22,720 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:22,720 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/26598bce8533482baaae8a17f0850ee3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/371af65bb2694edc8a971bd52c7e8dce, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/2664eb200a9c40ec9d188e070af90f51] 2024-12-09T06:17:22,720 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26598bce8533482baaae8a17f0850ee3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733725039531 2024-12-09T06:17:22,721 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 371af65bb2694edc8a971bd52c7e8dce, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733725039676 2024-12-09T06:17:22,721 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 470660e3139f491fbe01f30c588d8231, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733725039531 2024-12-09T06:17:22,722 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2664eb200a9c40ec9d188e070af90f51, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733725041287 2024-12-09T06:17:22,722 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e1049b0208c4401bba15a1d402acd352, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733725039676 2024-12-09T06:17:22,726 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 69303300a49a4e97912454032b088702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733725041290 2024-12-09T06:17:22,747 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:22,747 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/aeb28f3c8cfd45258f6a126cbd3ca98c is 50, key is test_row_0/B:col10/1733725041303/Put/seqid=0 2024-12-09T06:17:22,749 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:22,766 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120914b606810f8d48799ed2302d6f8b8643_1f2d999c565a1d93300e02727b70ded5 store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:22,768 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120914b606810f8d48799ed2302d6f8b8643_1f2d999c565a1d93300e02727b70ded5, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:22,768 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120914b606810f8d48799ed2302d6f8b8643_1f2d999c565a1d93300e02727b70ded5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:22,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742231_1407 (size=13085) 2024-12-09T06:17:22,806 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/aeb28f3c8cfd45258f6a126cbd3ca98c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/aeb28f3c8cfd45258f6a126cbd3ca98c 2024-12-09T06:17:22,813 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/B of 1f2d999c565a1d93300e02727b70ded5 into aeb28f3c8cfd45258f6a126cbd3ca98c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:22,813 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:22,813 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/B, priority=13, startTime=1733725042715; duration=0sec 2024-12-09T06:17:22,813 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:22,813 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:B 2024-12-09T06:17:22,813 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:22,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742232_1408 (size=4469) 2024-12-09T06:17:22,815 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:22,815 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/C is initiating minor compaction (all files) 2024-12-09T06:17:22,816 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/C in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:22,816 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/eb1dbba70a1f4827b71f09319d7240ad, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4f12b54a46f042f198ec5c54c1717860, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/802d1ffb3c1b4033a8d6e9f83ed14b90] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=34.3 K 2024-12-09T06:17:22,817 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#A#compaction#343 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:22,818 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/15bfc4c860b746e18026938f8e11ee06 is 175, key is test_row_0/A:col10/1733725041303/Put/seqid=0 2024-12-09T06:17:22,821 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting eb1dbba70a1f4827b71f09319d7240ad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733725039531 2024-12-09T06:17:22,822 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f12b54a46f042f198ec5c54c1717860, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733725039676 2024-12-09T06:17:22,822 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 802d1ffb3c1b4033a8d6e9f83ed14b90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733725041290 2024-12-09T06:17:22,836 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#C#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:22,837 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/705c006cd17b4d698a367ad95d6dfb37 is 50, key is test_row_0/C:col10/1733725041303/Put/seqid=0 2024-12-09T06:17:22,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742233_1409 (size=32039) 2024-12-09T06:17:22,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742234_1410 (size=13085) 2024-12-09T06:17:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T06:17:23,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:23,098 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-09T06:17:23,106 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:23,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:23,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T06:17:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T06:17:23,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T06:17:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:23,260 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T06:17:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:23,282 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/15bfc4c860b746e18026938f8e11ee06 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/15bfc4c860b746e18026938f8e11ee06 2024-12-09T06:17:23,292 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/A of 1f2d999c565a1d93300e02727b70ded5 into 15bfc4c860b746e18026938f8e11ee06(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:23,292 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:23,292 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/A, priority=13, startTime=1733725042715; duration=0sec 2024-12-09T06:17:23,292 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:23,292 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:A 2024-12-09T06:17:23,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c1d8e90226b04d00970b4542c1c5d6f3_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725041403/Put/seqid=0 2024-12-09T06:17:23,299 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/705c006cd17b4d698a367ad95d6dfb37 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/705c006cd17b4d698a367ad95d6dfb37 2024-12-09T06:17:23,321 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/C of 1f2d999c565a1d93300e02727b70ded5 into 705c006cd17b4d698a367ad95d6dfb37(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:23,321 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:23,321 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/C, priority=13, startTime=1733725042715; duration=0sec 2024-12-09T06:17:23,322 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:23,322 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:C 2024-12-09T06:17:23,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742235_1411 (size=12454) 2024-12-09T06:17:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:23,349 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c1d8e90226b04d00970b4542c1c5d6f3_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c1d8e90226b04d00970b4542c1c5d6f3_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:23,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/e06e38fd53bd4e36ad099c2f866543e6, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:23,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/e06e38fd53bd4e36ad099c2f866543e6 is 175, key is test_row_0/A:col10/1733725041403/Put/seqid=0 2024-12-09T06:17:23,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742236_1412 (size=31255) 2024-12-09T06:17:23,411 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=740, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/e06e38fd53bd4e36ad099c2f866543e6 2024-12-09T06:17:23,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/f59d5aebc8ae4227971941b2fb335d9c is 50, key is test_row_0/B:col10/1733725041403/Put/seqid=0 2024-12-09T06:17:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T06:17:23,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742237_1413 (size=12301) 2024-12-09T06:17:23,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. as already flushing 2024-12-09T06:17:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725103637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:23,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725103641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:23,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725103666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725103666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T06:17:23,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59704 deadline: 1733725103767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:23,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59774 deadline: 1733725103788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:23,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59730 deadline: 1733725103797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59768 deadline: 1733725103797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:23,816 INFO [Thread-1458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1458. 2024-12-09T06:17:23,816 DEBUG [Thread-1458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:23,816 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,816 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:23,823 INFO [Thread-1454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1454. 2024-12-09T06:17:23,823 DEBUG [Thread-1454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:23,824 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,824 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:23,827 INFO [Thread-1456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1456. 2024-12-09T06:17:23,827 DEBUG [Thread-1456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:23,827 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,827 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:23,833 INFO [Thread-1460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1460. 2024-12-09T06:17:23,833 DEBUG [Thread-1460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:23,833 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,833 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:23,836 INFO [Thread-1452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1452. 2024-12-09T06:17:23,836 DEBUG [Thread-1452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:23,836 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,837 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:23,874 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=740 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/f59d5aebc8ae4227971941b2fb335d9c 2024-12-09T06:17:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/0f286f23380846ccaaa6ca15aec48b99 is 50, key is test_row_0/C:col10/1733725041403/Put/seqid=0 2024-12-09T06:17:23,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742238_1414 (size=12301) 2024-12-09T06:17:23,893 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=740 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/0f286f23380846ccaaa6ca15aec48b99 2024-12-09T06:17:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/e06e38fd53bd4e36ad099c2f866543e6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e06e38fd53bd4e36ad099c2f866543e6 2024-12-09T06:17:23,902 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e06e38fd53bd4e36ad099c2f866543e6, entries=150, sequenceid=740, filesize=30.5 K 2024-12-09T06:17:23,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/f59d5aebc8ae4227971941b2fb335d9c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/f59d5aebc8ae4227971941b2fb335d9c 2024-12-09T06:17:23,910 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/f59d5aebc8ae4227971941b2fb335d9c, entries=150, sequenceid=740, filesize=12.0 K 2024-12-09T06:17:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/0f286f23380846ccaaa6ca15aec48b99 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0f286f23380846ccaaa6ca15aec48b99 2024-12-09T06:17:23,913 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0f286f23380846ccaaa6ca15aec48b99, entries=150, sequenceid=740, filesize=12.0 K 2024-12-09T06:17:23,914 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1f2d999c565a1d93300e02727b70ded5 in 654ms, sequenceid=740, compaction requested=false 2024-12-09T06:17:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:23,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-09T06:17:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-09T06:17:23,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-09T06:17:23,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 808 msec 2024-12-09T06:17:23,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 819 msec 2024-12-09T06:17:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:23,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T06:17:23,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:23,978 INFO [Thread-1449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1449. 2024-12-09T06:17:23,978 DEBUG [Thread-1449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:23,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:23,979 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:23,979 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:23,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:23,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096c737d4f1ffd4bce86418180003bca11_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_0/A:col10/1733725043977/Put/seqid=0 2024-12-09T06:17:24,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742239_1415 (size=12454) 2024-12-09T06:17:24,009 INFO [Thread-1447 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1447. 2024-12-09T06:17:24,009 DEBUG [Thread-1447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:24,009 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:24,009 DEBUG [Thread-1447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:24,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:24,009 INFO [Thread-1445 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1445. 2024-12-09T06:17:24,009 DEBUG [Thread-1445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:24,009 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:24,009 DEBUG [Thread-1445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:24,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:24,010 INFO [Thread-1443 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1443. 2024-12-09T06:17:24,010 DEBUG [Thread-1443 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:24,010 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:24,010 DEBUG [Thread-1443 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:24,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T06:17:24,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:24,401 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:24,404 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096c737d4f1ffd4bce86418180003bca11_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c737d4f1ffd4bce86418180003bca11_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:24,405 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/1627842fcf8c4e5fa30470da43a6568d, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:24,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/1627842fcf8c4e5fa30470da43a6568d is 175, key is test_row_0/A:col10/1733725043977/Put/seqid=0 2024-12-09T06:17:24,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742240_1416 (size=31255) 2024-12-09T06:17:24,809 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=757, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/1627842fcf8c4e5fa30470da43a6568d 2024-12-09T06:17:24,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c7b84f1d06b145699d9fd5acbfed471a is 50, key is test_row_0/B:col10/1733725043977/Put/seqid=0 2024-12-09T06:17:24,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742241_1417 (size=12301) 2024-12-09T06:17:24,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=757 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c7b84f1d06b145699d9fd5acbfed471a 2024-12-09T06:17:24,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/254685f108584c929823d36ba4d11e0c is 50, key is test_row_0/C:col10/1733725043977/Put/seqid=0 2024-12-09T06:17:24,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742242_1418 (size=12301) 2024-12-09T06:17:25,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=757 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/254685f108584c929823d36ba4d11e0c 2024-12-09T06:17:25,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/1627842fcf8c4e5fa30470da43a6568d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/1627842fcf8c4e5fa30470da43a6568d 2024-12-09T06:17:25,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/1627842fcf8c4e5fa30470da43a6568d, entries=150, sequenceid=757, filesize=30.5 K 2024-12-09T06:17:25,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/c7b84f1d06b145699d9fd5acbfed471a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c7b84f1d06b145699d9fd5acbfed471a 2024-12-09T06:17:25,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c7b84f1d06b145699d9fd5acbfed471a, entries=150, sequenceid=757, filesize=12.0 K 2024-12-09T06:17:25,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/254685f108584c929823d36ba4d11e0c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/254685f108584c929823d36ba4d11e0c 2024-12-09T06:17:25,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/254685f108584c929823d36ba4d11e0c, entries=150, sequenceid=757, filesize=12.0 K 2024-12-09T06:17:25,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for 1f2d999c565a1d93300e02727b70ded5 in 1268ms, sequenceid=757, compaction requested=true 2024-12-09T06:17:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:25,247 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:25,247 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f2d999c565a1d93300e02727b70ded5:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:25,247 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:25,247 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:25,247 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/B is initiating minor compaction (all files) 2024-12-09T06:17:25,247 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/A is initiating minor compaction (all files) 2024-12-09T06:17:25,247 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/B in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:25,247 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/A in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:25,248 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/aeb28f3c8cfd45258f6a126cbd3ca98c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/f59d5aebc8ae4227971941b2fb335d9c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c7b84f1d06b145699d9fd5acbfed471a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=36.8 K 2024-12-09T06:17:25,248 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/15bfc4c860b746e18026938f8e11ee06, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e06e38fd53bd4e36ad099c2f866543e6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/1627842fcf8c4e5fa30470da43a6568d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=92.3 K 2024-12-09T06:17:25,248 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:25,248 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/15bfc4c860b746e18026938f8e11ee06, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e06e38fd53bd4e36ad099c2f866543e6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/1627842fcf8c4e5fa30470da43a6568d] 2024-12-09T06:17:25,248 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb28f3c8cfd45258f6a126cbd3ca98c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733725041290 2024-12-09T06:17:25,248 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15bfc4c860b746e18026938f8e11ee06, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733725041290 2024-12-09T06:17:25,248 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06e38fd53bd4e36ad099c2f866543e6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=740, earliestPutTs=1733725041401 2024-12-09T06:17:25,248 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f59d5aebc8ae4227971941b2fb335d9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=740, earliestPutTs=1733725041401 2024-12-09T06:17:25,248 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1627842fcf8c4e5fa30470da43a6568d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=757, earliestPutTs=1733725043629 2024-12-09T06:17:25,248 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c7b84f1d06b145699d9fd5acbfed471a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=757, earliestPutTs=1733725043629 2024-12-09T06:17:25,255 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:25,255 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/b341f02c7098431c8cff0072a6dea34f is 50, key is test_row_0/B:col10/1733725043977/Put/seqid=0 2024-12-09T06:17:25,260 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:25,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742243_1419 (size=13187) 2024-12-09T06:17:25,265 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120987db4f2bf0fa4fdbb2c1966d6f4be3a8_1f2d999c565a1d93300e02727b70ded5 store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:25,266 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/b341f02c7098431c8cff0072a6dea34f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/b341f02c7098431c8cff0072a6dea34f 2024-12-09T06:17:25,270 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/B of 1f2d999c565a1d93300e02727b70ded5 into b341f02c7098431c8cff0072a6dea34f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:25,270 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:25,270 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/B, priority=13, startTime=1733725045247; duration=0sec 2024-12-09T06:17:25,270 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:25,270 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:B 2024-12-09T06:17:25,271 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:25,287 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120987db4f2bf0fa4fdbb2c1966d6f4be3a8_1f2d999c565a1d93300e02727b70ded5, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:25,287 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120987db4f2bf0fa4fdbb2c1966d6f4be3a8_1f2d999c565a1d93300e02727b70ded5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:25,289 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:25,289 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 1f2d999c565a1d93300e02727b70ded5/C is initiating minor compaction (all files) 2024-12-09T06:17:25,289 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f2d999c565a1d93300e02727b70ded5/C in TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:25,290 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/705c006cd17b4d698a367ad95d6dfb37, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0f286f23380846ccaaa6ca15aec48b99, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/254685f108584c929823d36ba4d11e0c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp, totalSize=36.8 K 2024-12-09T06:17:25,290 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 705c006cd17b4d698a367ad95d6dfb37, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733725041290 2024-12-09T06:17:25,291 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f286f23380846ccaaa6ca15aec48b99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=740, earliestPutTs=1733725041401 2024-12-09T06:17:25,291 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 254685f108584c929823d36ba4d11e0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=757, earliestPutTs=1733725043629 2024-12-09T06:17:25,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742244_1420 (size=4469) 2024-12-09T06:17:25,320 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#A#compaction#352 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:25,320 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/10071fd3a17747b98260ce68db815d37 is 175, key is test_row_0/A:col10/1733725043977/Put/seqid=0 2024-12-09T06:17:25,322 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f2d999c565a1d93300e02727b70ded5#C#compaction#353 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:25,323 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/fe5eeca01f3944758e22828a68067b29 is 50, key is test_row_0/C:col10/1733725043977/Put/seqid=0 2024-12-09T06:17:25,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742245_1421 (size=32141) 2024-12-09T06:17:25,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742246_1422 (size=13187) 2024-12-09T06:17:25,389 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/10071fd3a17747b98260ce68db815d37 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/10071fd3a17747b98260ce68db815d37 2024-12-09T06:17:25,396 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/A of 1f2d999c565a1d93300e02727b70ded5 into 10071fd3a17747b98260ce68db815d37(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:25,396 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:25,396 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/A, priority=13, startTime=1733725045247; duration=0sec 2024-12-09T06:17:25,396 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:25,396 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:A 2024-12-09T06:17:25,796 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/fe5eeca01f3944758e22828a68067b29 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fe5eeca01f3944758e22828a68067b29 2024-12-09T06:17:25,805 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f2d999c565a1d93300e02727b70ded5/C of 1f2d999c565a1d93300e02727b70ded5 into fe5eeca01f3944758e22828a68067b29(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:25,805 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f2d999c565a1d93300e02727b70ded5: 2024-12-09T06:17:25,805 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., storeName=1f2d999c565a1d93300e02727b70ded5/C, priority=13, startTime=1733725045247; duration=0sec 2024-12-09T06:17:25,805 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:25,805 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f2d999c565a1d93300e02727b70ded5:C 2024-12-09T06:17:26,609 INFO [Thread-1441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1441. 2024-12-09T06:17:26,609 DEBUG [Thread-1441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:26,609 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:26,609 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:26,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 188 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 196 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1133 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3397 rows 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1127 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3375 rows 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1119 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3356 rows 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1120 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3357 rows 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1119 2024-12-09T06:17:26,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3354 rows 2024-12-09T06:17:26,610 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T06:17:26,610 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:17:26,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:26,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:26,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:26,611 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T06:17:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T06:17:26,615 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725046615"}]},"ts":"1733725046615"} 2024-12-09T06:17:26,617 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T06:17:26,617 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T06:17:26,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:17:26,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, UNASSIGN}] 2024-12-09T06:17:26,620 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, UNASSIGN 2024-12-09T06:17:26,621 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=1f2d999c565a1d93300e02727b70ded5, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:26,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, UNASSIGN because future has completed 2024-12-09T06:17:26,623 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:17:26,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T06:17:26,775 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:26,775 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:17:26,776 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 1f2d999c565a1d93300e02727b70ded5, disabling compactions & flushes 2024-12-09T06:17:26,776 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:26,776 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:26,776 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. after waiting 0 ms 2024-12-09T06:17:26,776 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:26,776 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 1f2d999c565a1d93300e02727b70ded5 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T06:17:26,776 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=A 2024-12-09T06:17:26,776 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:26,777 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=B 2024-12-09T06:17:26,777 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:26,777 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1f2d999c565a1d93300e02727b70ded5, store=C 2024-12-09T06:17:26,777 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:26,786 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099d3217cdcc3b48f8b56828a44231b57a_1f2d999c565a1d93300e02727b70ded5 is 50, key is test_row_1/A:col10/1733725046607/Put/seqid=0 2024-12-09T06:17:26,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742247_1423 (size=9914) 2024-12-09T06:17:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T06:17:27,191 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:27,201 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099d3217cdcc3b48f8b56828a44231b57a_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d3217cdcc3b48f8b56828a44231b57a_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:27,202 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/0cc983048f484eec9afa6082eea9a674, store: [table=TestAcidGuarantees family=A region=1f2d999c565a1d93300e02727b70ded5] 2024-12-09T06:17:27,202 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/0cc983048f484eec9afa6082eea9a674 is 175, key is test_row_1/A:col10/1733725046607/Put/seqid=0 2024-12-09T06:17:27,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742248_1424 (size=22561) 2024-12-09T06:17:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T06:17:27,606 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=767, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/0cc983048f484eec9afa6082eea9a674 2024-12-09T06:17:27,612 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/485639be5da241daa1ec19f94cca09da is 50, key is test_row_1/B:col10/1733725046607/Put/seqid=0 2024-12-09T06:17:27,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742249_1425 (size=9857) 2024-12-09T06:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T06:17:28,015 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/485639be5da241daa1ec19f94cca09da 2024-12-09T06:17:28,021 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/d30eaccfdfbb4f10a7866118dfbacaac is 50, key is test_row_1/C:col10/1733725046607/Put/seqid=0 2024-12-09T06:17:28,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742250_1426 (size=9857) 2024-12-09T06:17:28,220 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T06:17:28,425 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/d30eaccfdfbb4f10a7866118dfbacaac 2024-12-09T06:17:28,440 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/A/0cc983048f484eec9afa6082eea9a674 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0cc983048f484eec9afa6082eea9a674 2024-12-09T06:17:28,445 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0cc983048f484eec9afa6082eea9a674, entries=100, sequenceid=767, filesize=22.0 K 2024-12-09T06:17:28,449 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/B/485639be5da241daa1ec19f94cca09da as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/485639be5da241daa1ec19f94cca09da 2024-12-09T06:17:28,453 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/485639be5da241daa1ec19f94cca09da, entries=100, sequenceid=767, filesize=9.6 K 2024-12-09T06:17:28,454 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/.tmp/C/d30eaccfdfbb4f10a7866118dfbacaac as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/d30eaccfdfbb4f10a7866118dfbacaac 2024-12-09T06:17:28,458 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/d30eaccfdfbb4f10a7866118dfbacaac, entries=100, sequenceid=767, filesize=9.6 K 2024-12-09T06:17:28,459 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1f2d999c565a1d93300e02727b70ded5 in 1683ms, sequenceid=767, compaction requested=false 2024-12-09T06:17:28,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/8623b8d1acdc42879f486a8730ea1124, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/80c7d11d69cc456bb9600dbca28dc35b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e1258c50f5824afa8ebfb331cb1ce178, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0c8ad172659400190e8a961c8db2254, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/6a810fb14d744904897bd1e618783a90, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/79c017bdcb35432fa4936353c78a1a31, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/049a0a403c174ab29db4f77759908a3f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5366220415534be28f14d3cca652c625, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/415d8e1f02fd4ed5ad65d93fe21882d2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0666f491e76f45c49abb554c2861148d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/68dcb971703742c7a3cd55b3f275a25f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/9df8b242ae2e4995b2b84342dda3da8a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/515569f4d6c94b6fac6d275044056352, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0d4687c5dc845adb9e31a58234a3253, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/b080a52ed1b14e8e81aa2e305214cf98, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5a9d9c542edd42c2bbbbaab83a892348, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/3ee947ef2f0b42b48362496ce960531a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/338be9d6e9244ca4831c008330df4a6b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/4498451d0b894efa9b84d1c27e22a7b6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/280c6f539c0141a19db78c279a8f5e08, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/26598bce8533482baaae8a17f0850ee3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/371af65bb2694edc8a971bd52c7e8dce, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/2664eb200a9c40ec9d188e070af90f51, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/15bfc4c860b746e18026938f8e11ee06, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e06e38fd53bd4e36ad099c2f866543e6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/1627842fcf8c4e5fa30470da43a6568d] to archive 2024-12-09T06:17:28,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:28,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/8623b8d1acdc42879f486a8730ea1124 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/8623b8d1acdc42879f486a8730ea1124 2024-12-09T06:17:28,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/80c7d11d69cc456bb9600dbca28dc35b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/80c7d11d69cc456bb9600dbca28dc35b 2024-12-09T06:17:28,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e1258c50f5824afa8ebfb331cb1ce178 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e1258c50f5824afa8ebfb331cb1ce178 2024-12-09T06:17:28,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0c8ad172659400190e8a961c8db2254 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0c8ad172659400190e8a961c8db2254 2024-12-09T06:17:28,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/6a810fb14d744904897bd1e618783a90 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/6a810fb14d744904897bd1e618783a90 2024-12-09T06:17:28,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/79c017bdcb35432fa4936353c78a1a31 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/79c017bdcb35432fa4936353c78a1a31 2024-12-09T06:17:28,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/049a0a403c174ab29db4f77759908a3f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/049a0a403c174ab29db4f77759908a3f 2024-12-09T06:17:28,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5366220415534be28f14d3cca652c625 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5366220415534be28f14d3cca652c625 2024-12-09T06:17:28,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/415d8e1f02fd4ed5ad65d93fe21882d2 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/415d8e1f02fd4ed5ad65d93fe21882d2 2024-12-09T06:17:28,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0666f491e76f45c49abb554c2861148d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0666f491e76f45c49abb554c2861148d 2024-12-09T06:17:28,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/68dcb971703742c7a3cd55b3f275a25f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/68dcb971703742c7a3cd55b3f275a25f 2024-12-09T06:17:28,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/9df8b242ae2e4995b2b84342dda3da8a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/9df8b242ae2e4995b2b84342dda3da8a 2024-12-09T06:17:28,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/515569f4d6c94b6fac6d275044056352 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/515569f4d6c94b6fac6d275044056352 2024-12-09T06:17:28,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0d4687c5dc845adb9e31a58234a3253 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/c0d4687c5dc845adb9e31a58234a3253 2024-12-09T06:17:28,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/b080a52ed1b14e8e81aa2e305214cf98 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/b080a52ed1b14e8e81aa2e305214cf98 2024-12-09T06:17:28,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5a9d9c542edd42c2bbbbaab83a892348 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/5a9d9c542edd42c2bbbbaab83a892348 2024-12-09T06:17:28,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/3ee947ef2f0b42b48362496ce960531a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/3ee947ef2f0b42b48362496ce960531a 2024-12-09T06:17:28,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/338be9d6e9244ca4831c008330df4a6b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/338be9d6e9244ca4831c008330df4a6b 2024-12-09T06:17:28,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/4498451d0b894efa9b84d1c27e22a7b6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/4498451d0b894efa9b84d1c27e22a7b6 2024-12-09T06:17:28,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/280c6f539c0141a19db78c279a8f5e08 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/280c6f539c0141a19db78c279a8f5e08 2024-12-09T06:17:28,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/26598bce8533482baaae8a17f0850ee3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/26598bce8533482baaae8a17f0850ee3 2024-12-09T06:17:28,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/371af65bb2694edc8a971bd52c7e8dce to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/371af65bb2694edc8a971bd52c7e8dce 2024-12-09T06:17:28,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/2664eb200a9c40ec9d188e070af90f51 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/2664eb200a9c40ec9d188e070af90f51 2024-12-09T06:17:28,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/15bfc4c860b746e18026938f8e11ee06 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/15bfc4c860b746e18026938f8e11ee06 2024-12-09T06:17:28,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e06e38fd53bd4e36ad099c2f866543e6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/e06e38fd53bd4e36ad099c2f866543e6 2024-12-09T06:17:28,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/1627842fcf8c4e5fa30470da43a6568d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/1627842fcf8c4e5fa30470da43a6568d 2024-12-09T06:17:28,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c27e65fd30644a6d88c3c4beafe72663, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/902946ee822e40a88348434c0293ef3b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a4783ff8893847609c7af96f2a422dd6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/2e633ba42a2a4d9dbf81ac2fbc452c79, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c15d9c196281440f88204d9d100126e8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/46fb74ab242f4cc58c10cb41faa55ccb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ea0d3dd61da44d799b91f8e820735a75, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/98ddd7d75afd4a62853c5ac32d0663af, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/3fc126140eae47978bbc600a8a16a36e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/12d6d215fb364af3963596b4003721a0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/26ab8576061f4f8e92642f86cbc38823, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ee0a28aba63c4e4e95423a724746ffbf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/cc86bb3352414a46b1cf137fac93161e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d6922ed363434e8198db9f9261a7aac2, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/134aa0edba624032bb7d4c856d5c64bf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4c107f11ea824432bfc23a60293ac4ce, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a31068bdd56b47f9a3a9dace42123d6d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4a119881a111466f9f51aad32dd55df7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a09075fcc2f440658c56c1642db0304d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/470660e3139f491fbe01f30c588d8231, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d505eeefea6d473aa4c20718a3fccc2e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/e1049b0208c4401bba15a1d402acd352, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/aeb28f3c8cfd45258f6a126cbd3ca98c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/69303300a49a4e97912454032b088702, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/f59d5aebc8ae4227971941b2fb335d9c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c7b84f1d06b145699d9fd5acbfed471a] to archive 2024-12-09T06:17:28,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:28,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c27e65fd30644a6d88c3c4beafe72663 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c27e65fd30644a6d88c3c4beafe72663 2024-12-09T06:17:28,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/902946ee822e40a88348434c0293ef3b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/902946ee822e40a88348434c0293ef3b 2024-12-09T06:17:28,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a4783ff8893847609c7af96f2a422dd6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a4783ff8893847609c7af96f2a422dd6 2024-12-09T06:17:28,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/2e633ba42a2a4d9dbf81ac2fbc452c79 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/2e633ba42a2a4d9dbf81ac2fbc452c79 2024-12-09T06:17:28,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c15d9c196281440f88204d9d100126e8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c15d9c196281440f88204d9d100126e8 2024-12-09T06:17:28,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/46fb74ab242f4cc58c10cb41faa55ccb to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/46fb74ab242f4cc58c10cb41faa55ccb 2024-12-09T06:17:28,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ea0d3dd61da44d799b91f8e820735a75 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ea0d3dd61da44d799b91f8e820735a75 2024-12-09T06:17:28,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/98ddd7d75afd4a62853c5ac32d0663af to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/98ddd7d75afd4a62853c5ac32d0663af 2024-12-09T06:17:28,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/3fc126140eae47978bbc600a8a16a36e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/3fc126140eae47978bbc600a8a16a36e 2024-12-09T06:17:28,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/12d6d215fb364af3963596b4003721a0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/12d6d215fb364af3963596b4003721a0 2024-12-09T06:17:28,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/26ab8576061f4f8e92642f86cbc38823 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/26ab8576061f4f8e92642f86cbc38823 2024-12-09T06:17:28,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ee0a28aba63c4e4e95423a724746ffbf to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/ee0a28aba63c4e4e95423a724746ffbf 2024-12-09T06:17:28,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/cc86bb3352414a46b1cf137fac93161e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/cc86bb3352414a46b1cf137fac93161e 2024-12-09T06:17:28,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d6922ed363434e8198db9f9261a7aac2 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d6922ed363434e8198db9f9261a7aac2 2024-12-09T06:17:28,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/134aa0edba624032bb7d4c856d5c64bf to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/134aa0edba624032bb7d4c856d5c64bf 2024-12-09T06:17:28,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4c107f11ea824432bfc23a60293ac4ce to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4c107f11ea824432bfc23a60293ac4ce 2024-12-09T06:17:28,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a31068bdd56b47f9a3a9dace42123d6d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a31068bdd56b47f9a3a9dace42123d6d 2024-12-09T06:17:28,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4a119881a111466f9f51aad32dd55df7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/4a119881a111466f9f51aad32dd55df7 2024-12-09T06:17:28,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a09075fcc2f440658c56c1642db0304d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/a09075fcc2f440658c56c1642db0304d 2024-12-09T06:17:28,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/470660e3139f491fbe01f30c588d8231 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/470660e3139f491fbe01f30c588d8231 2024-12-09T06:17:28,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d505eeefea6d473aa4c20718a3fccc2e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/d505eeefea6d473aa4c20718a3fccc2e 2024-12-09T06:17:28,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/e1049b0208c4401bba15a1d402acd352 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/e1049b0208c4401bba15a1d402acd352 2024-12-09T06:17:28,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/aeb28f3c8cfd45258f6a126cbd3ca98c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/aeb28f3c8cfd45258f6a126cbd3ca98c 2024-12-09T06:17:28,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/69303300a49a4e97912454032b088702 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/69303300a49a4e97912454032b088702 2024-12-09T06:17:28,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/f59d5aebc8ae4227971941b2fb335d9c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/f59d5aebc8ae4227971941b2fb335d9c 2024-12-09T06:17:28,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c7b84f1d06b145699d9fd5acbfed471a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/c7b84f1d06b145699d9fd5acbfed471a 2024-12-09T06:17:28,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e5168e7a9e624db796f86dd1e87a42f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/dcd0fe80d8dd429ba154f7885b7ba895, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/459a8af9731b4372b8f805979e5c40f9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e3df8cb8f8844c51a2f07a780d0e4670, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/91971c978c3f4e348a5700033491bdee, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/3f5054c3eee64bc3898ea320f3748968, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/98fd306416d0435c88833d8089867d89, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fa5e349adc6e4159b92e3ad33a6355d1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/07a75cfb4c7a41d786e3e8e56194c615, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/8a67f44eadc243c19208eeefc0d42b8b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/7e1b03b57a8f4aa49fde1a496007ea31, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/5134342abdcd4519911eb7c69d7de098, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2492137aa4c941b5b2328195f911efea, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4b264989fef94531944a071c5e66b3d4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/01e036eccb70409daae2d25b5cff2c5e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/6b9dd0be1464416cbb46469eb02fd947, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2966ddb6b9a242dd9893f29268dda9a0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/c862a414584741c494303bf963a69847, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/f9b1d35a9bf9423590124bc9a5e9c3a7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/eb1dbba70a1f4827b71f09319d7240ad, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0c9700b830f64ce4a6c277e570e98a7d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4f12b54a46f042f198ec5c54c1717860, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/705c006cd17b4d698a367ad95d6dfb37, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/802d1ffb3c1b4033a8d6e9f83ed14b90, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0f286f23380846ccaaa6ca15aec48b99, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/254685f108584c929823d36ba4d11e0c] to archive 2024-12-09T06:17:28,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:28,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e5168e7a9e624db796f86dd1e87a42f3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e5168e7a9e624db796f86dd1e87a42f3 2024-12-09T06:17:28,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/dcd0fe80d8dd429ba154f7885b7ba895 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/dcd0fe80d8dd429ba154f7885b7ba895 2024-12-09T06:17:28,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/459a8af9731b4372b8f805979e5c40f9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/459a8af9731b4372b8f805979e5c40f9 2024-12-09T06:17:28,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e3df8cb8f8844c51a2f07a780d0e4670 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/e3df8cb8f8844c51a2f07a780d0e4670 2024-12-09T06:17:28,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/91971c978c3f4e348a5700033491bdee to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/91971c978c3f4e348a5700033491bdee 2024-12-09T06:17:28,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/3f5054c3eee64bc3898ea320f3748968 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/3f5054c3eee64bc3898ea320f3748968 2024-12-09T06:17:28,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/98fd306416d0435c88833d8089867d89 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/98fd306416d0435c88833d8089867d89 2024-12-09T06:17:28,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fa5e349adc6e4159b92e3ad33a6355d1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fa5e349adc6e4159b92e3ad33a6355d1 2024-12-09T06:17:28,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/07a75cfb4c7a41d786e3e8e56194c615 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/07a75cfb4c7a41d786e3e8e56194c615 2024-12-09T06:17:28,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/8a67f44eadc243c19208eeefc0d42b8b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/8a67f44eadc243c19208eeefc0d42b8b 2024-12-09T06:17:28,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/7e1b03b57a8f4aa49fde1a496007ea31 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/7e1b03b57a8f4aa49fde1a496007ea31 2024-12-09T06:17:28,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/5134342abdcd4519911eb7c69d7de098 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/5134342abdcd4519911eb7c69d7de098 2024-12-09T06:17:28,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2492137aa4c941b5b2328195f911efea to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2492137aa4c941b5b2328195f911efea 2024-12-09T06:17:28,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4b264989fef94531944a071c5e66b3d4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4b264989fef94531944a071c5e66b3d4 2024-12-09T06:17:28,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/01e036eccb70409daae2d25b5cff2c5e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/01e036eccb70409daae2d25b5cff2c5e 2024-12-09T06:17:28,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/6b9dd0be1464416cbb46469eb02fd947 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/6b9dd0be1464416cbb46469eb02fd947 2024-12-09T06:17:28,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2966ddb6b9a242dd9893f29268dda9a0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/2966ddb6b9a242dd9893f29268dda9a0 2024-12-09T06:17:28,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/c862a414584741c494303bf963a69847 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/c862a414584741c494303bf963a69847 2024-12-09T06:17:28,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/f9b1d35a9bf9423590124bc9a5e9c3a7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/f9b1d35a9bf9423590124bc9a5e9c3a7 2024-12-09T06:17:28,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/eb1dbba70a1f4827b71f09319d7240ad to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/eb1dbba70a1f4827b71f09319d7240ad 2024-12-09T06:17:28,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0c9700b830f64ce4a6c277e570e98a7d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0c9700b830f64ce4a6c277e570e98a7d 2024-12-09T06:17:28,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4f12b54a46f042f198ec5c54c1717860 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/4f12b54a46f042f198ec5c54c1717860 2024-12-09T06:17:28,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/705c006cd17b4d698a367ad95d6dfb37 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/705c006cd17b4d698a367ad95d6dfb37 2024-12-09T06:17:28,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/802d1ffb3c1b4033a8d6e9f83ed14b90 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/802d1ffb3c1b4033a8d6e9f83ed14b90 2024-12-09T06:17:28,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0f286f23380846ccaaa6ca15aec48b99 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/0f286f23380846ccaaa6ca15aec48b99 2024-12-09T06:17:28,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/254685f108584c929823d36ba4d11e0c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/254685f108584c929823d36ba4d11e0c 2024-12-09T06:17:28,583 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/recovered.edits/770.seqid, newMaxSeqId=770, maxSeqId=4 2024-12-09T06:17:28,584 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. 2024-12-09T06:17:28,584 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 1f2d999c565a1d93300e02727b70ded5: Waiting for close lock at 1733725046775Running coprocessor pre-close hooks at 1733725046775Disabling compacts and flushes for region at 1733725046775Disabling writes for close at 1733725046776 (+1 ms)Obtaining lock to block concurrent updates at 1733725046776Preparing flush snapshotting stores in 1f2d999c565a1d93300e02727b70ded5 at 1733725046776Finished memstore snapshotting TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733725046777 (+1 ms)Flushing stores of TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5. at 1733725046778 (+1 ms)Flushing 1f2d999c565a1d93300e02727b70ded5/A: creating writer at 1733725046778Flushing mob file 1f2d999c565a1d93300e02727b70ded5/A: appending metadata at 1733725046786 (+8 ms)Flushing mob file 1f2d999c565a1d93300e02727b70ded5/A: closing flushed file at 1733725046786Flushing 1f2d999c565a1d93300e02727b70ded5/A: appending metadata at 1733725047202 (+416 ms)Flushing 1f2d999c565a1d93300e02727b70ded5/A: closing flushed file at 1733725047202Flushing 1f2d999c565a1d93300e02727b70ded5/B: creating writer at 1733725047609 (+407 ms)Flushing 1f2d999c565a1d93300e02727b70ded5/B: appending metadata at 1733725047611 (+2 ms)Flushing 1f2d999c565a1d93300e02727b70ded5/B: closing flushed file at 1733725047611Flushing 1f2d999c565a1d93300e02727b70ded5/C: creating writer at 1733725048019 (+408 ms)Flushing 1f2d999c565a1d93300e02727b70ded5/C: appending metadata at 1733725048021 (+2 ms)Flushing 1f2d999c565a1d93300e02727b70ded5/C: closing flushed file at 1733725048021Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@511ba44e: reopening flushed file at 1733725048439 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cad52f2: reopening flushed file at 1733725048445 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26938533: reopening flushed file at 1733725048453 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1f2d999c565a1d93300e02727b70ded5 in 1683ms, sequenceid=767, compaction requested=false at 1733725048459 (+6 ms)Writing region close event to WAL at 1733725048577 (+118 ms)Running coprocessor post-close hooks at 1733725048584 (+7 ms)Closed at 1733725048584 2024-12-09T06:17:28,587 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,587 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=1f2d999c565a1d93300e02727b70ded5, regionState=CLOSED 2024-12-09T06:17:28,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:28,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-09T06:17:28,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 1f2d999c565a1d93300e02727b70ded5, server=82ac79f0323b,42797,1733724930590 in 1.9750 sec 2024-12-09T06:17:28,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-09T06:17:28,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f2d999c565a1d93300e02727b70ded5, UNASSIGN in 1.9810 sec 2024-12-09T06:17:28,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-09T06:17:28,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9840 sec 2024-12-09T06:17:28,604 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725048604"}]},"ts":"1733725048604"} 2024-12-09T06:17:28,606 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T06:17:28,606 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T06:17:28,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9980 sec 2024-12-09T06:17:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T06:17:28,757 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:28,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T06:17:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:28,759 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T06:17:28,760 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:28,765 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,767 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/recovered.edits] 2024-12-09T06:17:28,769 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0cc983048f484eec9afa6082eea9a674 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/0cc983048f484eec9afa6082eea9a674 2024-12-09T06:17:28,770 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/10071fd3a17747b98260ce68db815d37 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/A/10071fd3a17747b98260ce68db815d37 2024-12-09T06:17:28,772 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/485639be5da241daa1ec19f94cca09da to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/485639be5da241daa1ec19f94cca09da 2024-12-09T06:17:28,773 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/b341f02c7098431c8cff0072a6dea34f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/B/b341f02c7098431c8cff0072a6dea34f 2024-12-09T06:17:28,776 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/d30eaccfdfbb4f10a7866118dfbacaac to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/d30eaccfdfbb4f10a7866118dfbacaac 2024-12-09T06:17:28,777 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fe5eeca01f3944758e22828a68067b29 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/C/fe5eeca01f3944758e22828a68067b29 2024-12-09T06:17:28,783 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/recovered.edits/770.seqid to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5/recovered.edits/770.seqid 2024-12-09T06:17:28,783 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,784 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T06:17:28,784 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T06:17:28,785 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T06:17:28,792 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120904c77a0a99804263b72e8b13cad15bec_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120904c77a0a99804263b72e8b13cad15bec_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,794 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090a49201fffd84781a549e7b06cfd5e9b_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090a49201fffd84781a549e7b06cfd5e9b_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,795 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090efad58528374c4289f19cd12aefc450_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412090efad58528374c4289f19cd12aefc450_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,796 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092b633788ccbb4cdda62f286e337b2d03_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092b633788ccbb4cdda62f286e337b2d03_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,797 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f0a10d6320c430d94087f6f0273d95f_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f0a10d6320c430d94087f6f0273d95f_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,799 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093b20248d7be24ec884227f450705ba00_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093b20248d7be24ec884227f450705ba00_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,801 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120957a4c6319c3842218e74ddbb0e80ee79_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120957a4c6319c3842218e74ddbb0e80ee79_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,802 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209624547aeeb1144ec8ffd30d0bc35a289_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209624547aeeb1144ec8ffd30d0bc35a289_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,803 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c737d4f1ffd4bce86418180003bca11_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096c737d4f1ffd4bce86418180003bca11_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,805 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097790059db16b409ba007d61756bb9219_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097790059db16b409ba007d61756bb9219_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,806 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120995b832f62f0c41c0bc4815d93ec3271c_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120995b832f62f0c41c0bc4815d93ec3271c_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,807 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120997be06ccf64547cb97906cbc73cb4fcb_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120997be06ccf64547cb97906cbc73cb4fcb_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,808 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ab74533999342a0a905a4c636627b3b_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099ab74533999342a0a905a4c636627b3b_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,809 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d3217cdcc3b48f8b56828a44231b57a_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d3217cdcc3b48f8b56828a44231b57a_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,810 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a8ca86e90eb642bca83272711c781ee2_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a8ca86e90eb642bca83272711c781ee2_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,811 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c1d8e90226b04d00970b4542c1c5d6f3_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c1d8e90226b04d00970b4542c1c5d6f3_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,812 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c92653a4a3cd427c99568e488b597a46_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c92653a4a3cd427c99568e488b597a46_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,813 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e34e7e2a1f8144bf806056cf4ad1afeb_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e34e7e2a1f8144bf806056cf4ad1afeb_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,814 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e4c85469d818428aa3fab4174458db9e_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e4c85469d818428aa3fab4174458db9e_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,815 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ffcf6036a5384d84b80e23742e95f0cf_1f2d999c565a1d93300e02727b70ded5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ffcf6036a5384d84b80e23742e95f0cf_1f2d999c565a1d93300e02727b70ded5 2024-12-09T06:17:28,815 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T06:17:28,817 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:28,819 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T06:17:28,822 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T06:17:28,823 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:28,823 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T06:17:28,823 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733725048823"}]},"ts":"9223372036854775807"} 2024-12-09T06:17:28,825 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T06:17:28,825 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1f2d999c565a1d93300e02727b70ded5, NAME => 'TestAcidGuarantees,,1733725020665.1f2d999c565a1d93300e02727b70ded5.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T06:17:28,825 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T06:17:28,825 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733725048825"}]},"ts":"9223372036854775807"} 2024-12-09T06:17:28,827 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T06:17:28,827 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:28,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-12-09T06:17:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T06:17:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T06:17:28,870 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:28,881 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 250), OpenFileDescriptor=463 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=919 (was 1026), ProcessCount=11 (was 11), AvailableMemoryMB=5373 (was 4784) - AvailableMemoryMB LEAK? - 2024-12-09T06:17:28,894 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=919, ProcessCount=11, AvailableMemoryMB=5372 2024-12-09T06:17:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:17:28,895 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:17:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:28,898 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T06:17:28,898 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:28,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-09T06:17:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T06:17:28,899 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T06:17:28,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742251_1427 (size=1061) 2024-12-09T06:17:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T06:17:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T06:17:29,317 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d 2024-12-09T06:17:29,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742252_1428 (size=53) 2024-12-09T06:17:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T06:17:29,729 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:17:29,729 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c69d9e9023ef26767a3f09a43f2c05f1, disabling compactions & flushes 2024-12-09T06:17:29,729 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:29,729 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:29,729 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. after waiting 0 ms 2024-12-09T06:17:29,729 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:29,729 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:29,729 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c69d9e9023ef26767a3f09a43f2c05f1: Waiting for close lock at 1733725049729Disabling compacts and flushes for region at 1733725049729Disabling writes for close at 1733725049729Writing region close event to WAL at 1733725049729Closed at 1733725049729 2024-12-09T06:17:29,730 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T06:17:29,730 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733725049730"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733725049730"}]},"ts":"1733725049730"} 2024-12-09T06:17:29,731 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T06:17:29,732 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T06:17:29,732 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725049732"}]},"ts":"1733725049732"} 2024-12-09T06:17:29,733 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T06:17:29,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, ASSIGN}] 2024-12-09T06:17:29,735 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, ASSIGN 2024-12-09T06:17:29,735 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, ASSIGN; state=OFFLINE, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=false 2024-12-09T06:17:29,881 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T06:17:29,886 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=c69d9e9023ef26767a3f09a43f2c05f1, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:29,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, ASSIGN because future has completed 2024-12-09T06:17:29,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T06:17:30,044 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:30,044 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:17:30,046 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,046 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:17:30,047 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,047 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,048 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,049 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:30,050 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c69d9e9023ef26767a3f09a43f2c05f1 columnFamilyName A 2024-12-09T06:17:30,050 DEBUG [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:30,050 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.HStore(327): Store=c69d9e9023ef26767a3f09a43f2c05f1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:30,050 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,051 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:30,051 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c69d9e9023ef26767a3f09a43f2c05f1 columnFamilyName B 2024-12-09T06:17:30,051 DEBUG [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:30,051 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.HStore(327): Store=c69d9e9023ef26767a3f09a43f2c05f1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:30,051 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,052 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:30,052 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c69d9e9023ef26767a3f09a43f2c05f1 columnFamilyName C 2024-12-09T06:17:30,052 DEBUG [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:30,053 INFO [StoreOpener-c69d9e9023ef26767a3f09a43f2c05f1-1 {}] regionserver.HStore(327): Store=c69d9e9023ef26767a3f09a43f2c05f1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:30,053 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:30,053 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,053 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,054 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,054 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,054 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,055 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:17:30,055 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,057 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:17:30,057 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened c69d9e9023ef26767a3f09a43f2c05f1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65667176, jitterRate=-0.02148282527923584}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:17:30,057 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:30,057 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for c69d9e9023ef26767a3f09a43f2c05f1: Running coprocessor pre-open hook at 1733725050047Writing region info on filesystem at 1733725050047Initializing all the Stores at 1733725050048 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725050048Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725050048Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725050048Cleaning up temporary data from old regions at 1733725050054 (+6 ms)Running coprocessor post-open hooks at 1733725050057 (+3 ms)Region opened successfully at 1733725050057 2024-12-09T06:17:30,058 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., pid=114, masterSystemTime=1733725050039 2024-12-09T06:17:30,059 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:30,059 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:30,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=c69d9e9023ef26767a3f09a43f2c05f1, regionState=OPEN, openSeqNum=2, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:30,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:30,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-09T06:17:30,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 in 174 msec 2024-12-09T06:17:30,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-09T06:17:30,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, ASSIGN in 328 msec 2024-12-09T06:17:30,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T06:17:30,065 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725050064"}]},"ts":"1733725050064"} 2024-12-09T06:17:30,066 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T06:17:30,066 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T06:17:30,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-12-09T06:17:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-09T06:17:31,049 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:31,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,053 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5e48d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,054 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f1a7d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,055 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,057 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T06:17:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 82ac79f0323b,43241,1733724929416 2024-12-09T06:17:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61f02ac6 2024-12-09T06:17:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T06:17:31,058 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T06:17:31,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,060 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc932a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,061 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7946f8f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,062 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcd7c0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46606b2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,065 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479e6a1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,066 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0e30b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,068 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4300be71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,069 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a58db7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,070 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,071 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20356edc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,072 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb7277e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,073 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,073 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35698b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,074 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8fe95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,076 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b43e5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f0c5e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,079 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5e2e2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,081 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9b9833, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,082 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21691029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,084 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7310f724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:31,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:31,085 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f61094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:31,087 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d8c2dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:31,088 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,088 DEBUG [Thread-1926 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,088 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,089 DEBUG [Thread-1937 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,089 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,089 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,090 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-09T06:17:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,091 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T06:17:31,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,092 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,093 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:31,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,094 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,095 DEBUG [Thread-1939 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,096 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2] 2024-12-09T06:17:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:31,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:17:31,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:31,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:31,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:31,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:31,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:31,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:31,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/92e145f87f2f40a98a5b459ecec718e6 is 50, key is test_row_0/A:col10/1733725051101/Put/seqid=0 2024-12-09T06:17:31,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742253_1429 (size=12001) 2024-12-09T06:17:31,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/92e145f87f2f40a98a5b459ecec718e6 2024-12-09T06:17:31,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725111138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725111138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725111140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725111142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/46560cfdc71f440a93d6f9b38c06a8c4 is 50, key is test_row_0/B:col10/1733725051101/Put/seqid=0 2024-12-09T06:17:31,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725111148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742254_1430 (size=12001) 2024-12-09T06:17:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T06:17:31,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T06:17:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725111256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725111257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725111257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725111257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725111257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T06:17:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:31,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T06:17:31,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725111466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725111466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725111466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725111466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725111466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T06:17:31,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:31,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/46560cfdc71f440a93d6f9b38c06a8c4 2024-12-09T06:17:31,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/e0731d4f264542bb9d1086a3582dae42 is 50, key is test_row_0/C:col10/1733725051101/Put/seqid=0 2024-12-09T06:17:31,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742255_1431 (size=12001) 2024-12-09T06:17:31,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T06:17:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T06:17:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725111777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725111777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725111777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725111777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725111777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:31,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T06:17:31,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:31,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:31,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:31,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/e0731d4f264542bb9d1086a3582dae42 2024-12-09T06:17:31,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/92e145f87f2f40a98a5b459ecec718e6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/92e145f87f2f40a98a5b459ecec718e6 2024-12-09T06:17:31,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/92e145f87f2f40a98a5b459ecec718e6, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T06:17:31,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/46560cfdc71f440a93d6f9b38c06a8c4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/46560cfdc71f440a93d6f9b38c06a8c4 2024-12-09T06:17:32,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/46560cfdc71f440a93d6f9b38c06a8c4, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T06:17:32,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/e0731d4f264542bb9d1086a3582dae42 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/e0731d4f264542bb9d1086a3582dae42 2024-12-09T06:17:32,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T06:17:32,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:32,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:32,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:32,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:32,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:32,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/e0731d4f264542bb9d1086a3582dae42, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T06:17:32,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c69d9e9023ef26767a3f09a43f2c05f1 in 913ms, sequenceid=13, compaction requested=false 2024-12-09T06:17:32,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:32,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T06:17:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:32,163 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T06:17:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:32,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/8c8fea3cb4e54c83bce9be2effc7fd61 is 50, key is test_row_0/A:col10/1733725051138/Put/seqid=0 2024-12-09T06:17:32,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742256_1432 (size=12001) 2024-12-09T06:17:32,188 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/8c8fea3cb4e54c83bce9be2effc7fd61 2024-12-09T06:17:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/723723d0165346beb6e01fa0f48d39c7 is 50, key is test_row_0/B:col10/1733725051138/Put/seqid=0 2024-12-09T06:17:32,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742257_1433 (size=12001) 2024-12-09T06:17:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T06:17:32,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:32,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725112295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725112296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725112297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725112302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725112303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725112406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725112407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725112417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725112417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725112418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,611 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/723723d0165346beb6e01fa0f48d39c7 2024-12-09T06:17:32,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/f7ef0ff788aa4cd794882f77ab1e4914 is 50, key is test_row_0/C:col10/1733725051138/Put/seqid=0 2024-12-09T06:17:32,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725112617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725112617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725112627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725112627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725112627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742258_1434 (size=12001) 2024-12-09T06:17:32,636 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/f7ef0ff788aa4cd794882f77ab1e4914 2024-12-09T06:17:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/8c8fea3cb4e54c83bce9be2effc7fd61 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8c8fea3cb4e54c83bce9be2effc7fd61 2024-12-09T06:17:32,664 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8c8fea3cb4e54c83bce9be2effc7fd61, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T06:17:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/723723d0165346beb6e01fa0f48d39c7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/723723d0165346beb6e01fa0f48d39c7 2024-12-09T06:17:32,669 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/723723d0165346beb6e01fa0f48d39c7, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T06:17:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/f7ef0ff788aa4cd794882f77ab1e4914 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f7ef0ff788aa4cd794882f77ab1e4914 2024-12-09T06:17:32,674 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f7ef0ff788aa4cd794882f77ab1e4914, entries=150, sequenceid=33, filesize=11.7 K 2024-12-09T06:17:32,675 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c69d9e9023ef26767a3f09a43f2c05f1 in 511ms, sequenceid=33, compaction requested=false 2024-12-09T06:17:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-09T06:17:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-09T06:17:32,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-09T06:17:32,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-12-09T06:17:32,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.5900 sec 2024-12-09T06:17:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:32,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T06:17:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:32,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/de8e1a8e02844ce89d816db69cf2abcf is 50, key is test_row_0/A:col10/1733725052941/Put/seqid=0 2024-12-09T06:17:32,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742259_1435 (size=16681) 2024-12-09T06:17:32,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725112962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725112963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725112964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725112965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:32,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725112969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:32,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:32,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725113077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725113077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725113077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725113077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725113077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T06:17:33,237 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:33,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-09T06:17:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T06:17:33,238 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:33,239 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:33,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725113287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725113287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725113287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725113287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725113288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T06:17:33,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/de8e1a8e02844ce89d816db69cf2abcf 2024-12-09T06:17:33,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9f98e0bf72d247e0a2777353bbc2ee93 is 50, key is test_row_0/B:col10/1733725052941/Put/seqid=0 2024-12-09T06:17:33,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742260_1436 (size=12001) 2024-12-09T06:17:33,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9f98e0bf72d247e0a2777353bbc2ee93 2024-12-09T06:17:33,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/59a45ffc62474d0bb7ee9319ce5b3229 is 50, key is test_row_0/C:col10/1733725052941/Put/seqid=0 2024-12-09T06:17:33,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T06:17:33,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:33,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742261_1437 (size=12001) 2024-12-09T06:17:33,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T06:17:33,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:33,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T06:17:33,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725113597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725113597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725113597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725113597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725113597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:33,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T06:17:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:33,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/59a45ffc62474d0bb7ee9319ce5b3229 2024-12-09T06:17:33,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/de8e1a8e02844ce89d816db69cf2abcf as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/de8e1a8e02844ce89d816db69cf2abcf 2024-12-09T06:17:33,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/de8e1a8e02844ce89d816db69cf2abcf, entries=250, sequenceid=47, filesize=16.3 K 2024-12-09T06:17:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9f98e0bf72d247e0a2777353bbc2ee93 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9f98e0bf72d247e0a2777353bbc2ee93 2024-12-09T06:17:33,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9f98e0bf72d247e0a2777353bbc2ee93, entries=150, sequenceid=47, filesize=11.7 K 2024-12-09T06:17:33,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/59a45ffc62474d0bb7ee9319ce5b3229 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/59a45ffc62474d0bb7ee9319ce5b3229 2024-12-09T06:17:33,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/59a45ffc62474d0bb7ee9319ce5b3229, entries=150, sequenceid=47, filesize=11.7 K 2024-12-09T06:17:33,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c69d9e9023ef26767a3f09a43f2c05f1 in 897ms, sequenceid=47, compaction requested=true 2024-12-09T06:17:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:33,838 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:33,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:33,839 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:33,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:33,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:33,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:33,839 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:33,839 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:33,839 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:33,839 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:33,840 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,840 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,840 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/92e145f87f2f40a98a5b459ecec718e6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8c8fea3cb4e54c83bce9be2effc7fd61, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/de8e1a8e02844ce89d816db69cf2abcf] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=39.7 K 2024-12-09T06:17:33,840 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/46560cfdc71f440a93d6f9b38c06a8c4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/723723d0165346beb6e01fa0f48d39c7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9f98e0bf72d247e0a2777353bbc2ee93] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.2 K 2024-12-09T06:17:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:33,840 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92e145f87f2f40a98a5b459ecec718e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733725051101 2024-12-09T06:17:33,840 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 46560cfdc71f440a93d6f9b38c06a8c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733725051101 2024-12-09T06:17:33,840 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8fea3cb4e54c83bce9be2effc7fd61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733725051137 2024-12-09T06:17:33,840 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 723723d0165346beb6e01fa0f48d39c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733725051137 2024-12-09T06:17:33,840 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting de8e1a8e02844ce89d816db69cf2abcf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733725052295 2024-12-09T06:17:33,841 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f98e0bf72d247e0a2777353bbc2ee93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733725052300 2024-12-09T06:17:33,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T06:17:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,854 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T06:17:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:33,855 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:33,855 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/ce7eb35e68034c839976b9e09fea2dc0 is 50, key is test_row_0/B:col10/1733725052941/Put/seqid=0 2024-12-09T06:17:33,858 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:33,859 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/62dec0fc70ad4ff3a3361b3bd6dcba07 is 50, key is test_row_0/A:col10/1733725052941/Put/seqid=0 2024-12-09T06:17:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T06:17:33,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0b5ef684380947d29a0435dbb32eba8f is 50, key is test_row_0/A:col10/1733725052962/Put/seqid=0 2024-12-09T06:17:33,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742263_1439 (size=12104) 2024-12-09T06:17:33,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742262_1438 (size=12104) 2024-12-09T06:17:33,875 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/62dec0fc70ad4ff3a3361b3bd6dcba07 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/62dec0fc70ad4ff3a3361b3bd6dcba07 2024-12-09T06:17:33,880 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/ce7eb35e68034c839976b9e09fea2dc0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ce7eb35e68034c839976b9e09fea2dc0 2024-12-09T06:17:33,882 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into 62dec0fc70ad4ff3a3361b3bd6dcba07(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:33,882 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:33,882 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725053838; duration=0sec 2024-12-09T06:17:33,882 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:33,882 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:33,882 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:33,884 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:33,884 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:33,884 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:33,884 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/e0731d4f264542bb9d1086a3582dae42, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f7ef0ff788aa4cd794882f77ab1e4914, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/59a45ffc62474d0bb7ee9319ce5b3229] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.2 K 2024-12-09T06:17:33,886 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0731d4f264542bb9d1086a3582dae42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733725051101 2024-12-09T06:17:33,886 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7ef0ff788aa4cd794882f77ab1e4914, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733725051137 2024-12-09T06:17:33,887 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59a45ffc62474d0bb7ee9319ce5b3229, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733725052300 2024-12-09T06:17:33,888 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into ce7eb35e68034c839976b9e09fea2dc0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:33,888 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:33,888 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725053839; duration=0sec 2024-12-09T06:17:33,889 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:33,889 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:33,911 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:33,911 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/4f43a71651434f1c8591d11f987c1d5b is 50, key is test_row_0/C:col10/1733725052941/Put/seqid=0 2024-12-09T06:17:33,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742264_1440 (size=12001) 2024-12-09T06:17:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742265_1441 (size=12104) 2024-12-09T06:17:33,972 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/4f43a71651434f1c8591d11f987c1d5b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/4f43a71651434f1c8591d11f987c1d5b 2024-12-09T06:17:33,981 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into 4f43a71651434f1c8591d11f987c1d5b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:33,981 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:33,981 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725053839; duration=0sec 2024-12-09T06:17:33,981 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:33,981 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:34,109 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T06:17:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:34,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725114123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725114126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725114126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725114128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725114134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725114237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725114247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725114247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725114247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725114247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,328 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0b5ef684380947d29a0435dbb32eba8f 2024-12-09T06:17:34,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9137af71e4d04593b0bf45db9059194b is 50, key is test_row_0/B:col10/1733725052962/Put/seqid=0 2024-12-09T06:17:34,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742266_1442 (size=12001) 2024-12-09T06:17:34,376 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9137af71e4d04593b0bf45db9059194b 2024-12-09T06:17:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T06:17:34,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/6df6b19076b44100b211df3d510958aa is 50, key is test_row_0/C:col10/1733725052962/Put/seqid=0 2024-12-09T06:17:34,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742267_1443 (size=12001) 2024-12-09T06:17:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725114446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725114457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725114457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725114459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725114466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,657 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/92e145f87f2f40a98a5b459ecec718e6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8c8fea3cb4e54c83bce9be2effc7fd61, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/de8e1a8e02844ce89d816db69cf2abcf] to archive 2024-12-09T06:17:34,658 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:34,660 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/92e145f87f2f40a98a5b459ecec718e6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/92e145f87f2f40a98a5b459ecec718e6 2024-12-09T06:17:34,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8c8fea3cb4e54c83bce9be2effc7fd61 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8c8fea3cb4e54c83bce9be2effc7fd61 2024-12-09T06:17:34,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/de8e1a8e02844ce89d816db69cf2abcf to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/de8e1a8e02844ce89d816db69cf2abcf 2024-12-09T06:17:34,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/46560cfdc71f440a93d6f9b38c06a8c4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/723723d0165346beb6e01fa0f48d39c7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9f98e0bf72d247e0a2777353bbc2ee93] to archive 2024-12-09T06:17:34,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:34,667 WARN [regionserver/82ac79f0323b:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-12-09T06:17:34,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/46560cfdc71f440a93d6f9b38c06a8c4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/46560cfdc71f440a93d6f9b38c06a8c4 2024-12-09T06:17:34,670 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/723723d0165346beb6e01fa0f48d39c7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/723723d0165346beb6e01fa0f48d39c7 2024-12-09T06:17:34,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9f98e0bf72d247e0a2777353bbc2ee93 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9f98e0bf72d247e0a2777353bbc2ee93 2024-12-09T06:17:34,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/e0731d4f264542bb9d1086a3582dae42, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f7ef0ff788aa4cd794882f77ab1e4914, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/59a45ffc62474d0bb7ee9319ce5b3229] to archive 2024-12-09T06:17:34,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:34,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/e0731d4f264542bb9d1086a3582dae42 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/e0731d4f264542bb9d1086a3582dae42 2024-12-09T06:17:34,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f7ef0ff788aa4cd794882f77ab1e4914 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f7ef0ff788aa4cd794882f77ab1e4914 2024-12-09T06:17:34,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/82ac79f0323b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/59a45ffc62474d0bb7ee9319ce5b3229 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/59a45ffc62474d0bb7ee9319ce5b3229 2024-12-09T06:17:34,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725114756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725114767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725114767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725114776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725114776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:34,798 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/6df6b19076b44100b211df3d510958aa 2024-12-09T06:17:34,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0b5ef684380947d29a0435dbb32eba8f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b5ef684380947d29a0435dbb32eba8f 2024-12-09T06:17:34,806 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b5ef684380947d29a0435dbb32eba8f, entries=150, sequenceid=65, filesize=11.7 K 2024-12-09T06:17:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9137af71e4d04593b0bf45db9059194b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9137af71e4d04593b0bf45db9059194b 2024-12-09T06:17:34,823 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9137af71e4d04593b0bf45db9059194b, entries=150, sequenceid=65, filesize=11.7 K 2024-12-09T06:17:34,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/6df6b19076b44100b211df3d510958aa as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/6df6b19076b44100b211df3d510958aa 2024-12-09T06:17:34,827 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/6df6b19076b44100b211df3d510958aa, entries=150, sequenceid=65, filesize=11.7 K 2024-12-09T06:17:34,828 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c69d9e9023ef26767a3f09a43f2c05f1 in 974ms, sequenceid=65, compaction requested=false 2024-12-09T06:17:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:34,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-09T06:17:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-09T06:17:34,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-09T06:17:34,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5930 sec 2024-12-09T06:17:34,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.5980 sec 2024-12-09T06:17:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:35,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T06:17:35,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:35,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:35,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:35,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:35,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:35,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:35,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/d912f878469f40ffb1ffc9d600398357 is 50, key is test_row_0/A:col10/1733725054127/Put/seqid=0 2024-12-09T06:17:35,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742268_1444 (size=14341) 2024-12-09T06:17:35,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725115302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725115303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725115303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725115312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725115313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T06:17:35,387 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:35,390 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-09T06:17:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T06:17:35,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:35,394 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:35,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:35,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725115416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725115417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725115417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725115437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725115437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T06:17:35,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T06:17:35,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:35,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:35,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:35,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725115627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725115627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725115627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725115646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725115656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/d912f878469f40ffb1ffc9d600398357 2024-12-09T06:17:35,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/88f3d1e4f12848d986f484dface19b14 is 50, key is test_row_0/B:col10/1733725054127/Put/seqid=0 2024-12-09T06:17:35,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T06:17:35,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:35,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:35,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:35,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742269_1445 (size=12001) 2024-12-09T06:17:35,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/88f3d1e4f12848d986f484dface19b14 2024-12-09T06:17:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T06:17:35,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/606ecfa7736c46beafae99e970358c58 is 50, key is test_row_0/C:col10/1733725054127/Put/seqid=0 2024-12-09T06:17:35,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742270_1446 (size=12001) 2024-12-09T06:17:35,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T06:17:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:35,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:35,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725115936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725115936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725115936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725115957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:35,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725115966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T06:17:36,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:36,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:36,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:36,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:36,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T06:17:36,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/606ecfa7736c46beafae99e970358c58 2024-12-09T06:17:36,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/d912f878469f40ffb1ffc9d600398357 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d912f878469f40ffb1ffc9d600398357 2024-12-09T06:17:36,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d912f878469f40ffb1ffc9d600398357, entries=200, sequenceid=83, filesize=14.0 K 2024-12-09T06:17:36,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/88f3d1e4f12848d986f484dface19b14 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/88f3d1e4f12848d986f484dface19b14 2024-12-09T06:17:36,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/88f3d1e4f12848d986f484dface19b14, entries=150, sequenceid=83, filesize=11.7 K 2024-12-09T06:17:36,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/606ecfa7736c46beafae99e970358c58 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/606ecfa7736c46beafae99e970358c58 2024-12-09T06:17:36,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/606ecfa7736c46beafae99e970358c58, entries=150, sequenceid=83, filesize=11.7 K 2024-12-09T06:17:36,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c69d9e9023ef26767a3f09a43f2c05f1 in 862ms, sequenceid=83, compaction requested=true 2024-12-09T06:17:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:36,143 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T06:17:36,143 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:36,144 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:36,144 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:36,144 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:36,144 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:36,144 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:36,144 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:36,144 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/62dec0fc70ad4ff3a3361b3bd6dcba07, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b5ef684380947d29a0435dbb32eba8f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d912f878469f40ffb1ffc9d600398357] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=37.5 K 2024-12-09T06:17:36,144 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ce7eb35e68034c839976b9e09fea2dc0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9137af71e4d04593b0bf45db9059194b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/88f3d1e4f12848d986f484dface19b14] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.3 K 2024-12-09T06:17:36,145 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62dec0fc70ad4ff3a3361b3bd6dcba07, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733725052300 2024-12-09T06:17:36,145 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7eb35e68034c839976b9e09fea2dc0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733725052300 2024-12-09T06:17:36,145 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9137af71e4d04593b0bf45db9059194b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725052952 2024-12-09T06:17:36,145 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b5ef684380947d29a0435dbb32eba8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725052952 2024-12-09T06:17:36,145 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f3d1e4f12848d986f484dface19b14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733725054127 2024-12-09T06:17:36,145 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting d912f878469f40ffb1ffc9d600398357, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733725054121 2024-12-09T06:17:36,158 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#375 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:36,159 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/ff63515bb8d44afb929f5e7479be0d31 is 50, key is test_row_0/A:col10/1733725054127/Put/seqid=0 2024-12-09T06:17:36,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T06:17:36,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:36,164 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#376 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:36,164 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T06:17:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:36,169 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/f02fec71447b4c58aa8cf392a5af885a is 50, key is test_row_0/B:col10/1733725054127/Put/seqid=0 2024-12-09T06:17:36,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742271_1447 (size=12104) 2024-12-09T06:17:36,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742272_1448 (size=12104) 2024-12-09T06:17:36,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0f0fb84c8ea24a91ae501d01497e30ad is 50, key is test_row_0/A:col10/1733725055302/Put/seqid=0 2024-12-09T06:17:36,194 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/f02fec71447b4c58aa8cf392a5af885a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f02fec71447b4c58aa8cf392a5af885a 2024-12-09T06:17:36,198 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into f02fec71447b4c58aa8cf392a5af885a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:36,198 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:36,198 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725056143; duration=0sec 2024-12-09T06:17:36,198 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:36,198 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:36,198 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:36,199 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:36,199 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:36,199 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:36,199 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/4f43a71651434f1c8591d11f987c1d5b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/6df6b19076b44100b211df3d510958aa, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/606ecfa7736c46beafae99e970358c58] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.3 K 2024-12-09T06:17:36,199 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f43a71651434f1c8591d11f987c1d5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733725052300 2024-12-09T06:17:36,201 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df6b19076b44100b211df3d510958aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725052952 2024-12-09T06:17:36,201 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 606ecfa7736c46beafae99e970358c58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733725054127 2024-12-09T06:17:36,210 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:36,210 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/d687b403bfa54417bcc01d1ccf66d252 is 50, key is test_row_0/C:col10/1733725054127/Put/seqid=0 2024-12-09T06:17:36,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742273_1449 (size=12001) 2024-12-09T06:17:36,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742274_1450 (size=12104) 2024-12-09T06:17:36,257 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/d687b403bfa54417bcc01d1ccf66d252 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/d687b403bfa54417bcc01d1ccf66d252 2024-12-09T06:17:36,262 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into d687b403bfa54417bcc01d1ccf66d252(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:36,262 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:36,262 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725056143; duration=0sec 2024-12-09T06:17:36,262 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:36,262 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:36,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:36,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725116467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725116472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725116473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725116477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725116477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T06:17:36,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725116577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725116586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725116586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,591 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/ff63515bb8d44afb929f5e7479be0d31 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ff63515bb8d44afb929f5e7479be0d31 2024-12-09T06:17:36,595 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into ff63515bb8d44afb929f5e7479be0d31(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:36,595 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:36,595 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725056143; duration=0sec 2024-12-09T06:17:36,595 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:36,595 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:36,623 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0f0fb84c8ea24a91ae501d01497e30ad 2024-12-09T06:17:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/64f5470ac28446c889ed1608502d1a7e is 50, key is test_row_0/B:col10/1733725055302/Put/seqid=0 2024-12-09T06:17:36,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742275_1451 (size=12001) 2024-12-09T06:17:36,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725116786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725116796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725116797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,043 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/64f5470ac28446c889ed1608502d1a7e 2024-12-09T06:17:37,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/3ae08c1fe5f74f73b55edbef74b923b6 is 50, key is test_row_0/C:col10/1733725055302/Put/seqid=0 2024-12-09T06:17:37,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742276_1452 (size=12001) 2024-12-09T06:17:37,065 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/3ae08c1fe5f74f73b55edbef74b923b6 2024-12-09T06:17:37,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0f0fb84c8ea24a91ae501d01497e30ad as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0f0fb84c8ea24a91ae501d01497e30ad 2024-12-09T06:17:37,080 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0f0fb84c8ea24a91ae501d01497e30ad, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T06:17:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/64f5470ac28446c889ed1608502d1a7e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/64f5470ac28446c889ed1608502d1a7e 2024-12-09T06:17:37,085 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/64f5470ac28446c889ed1608502d1a7e, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T06:17:37,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/3ae08c1fe5f74f73b55edbef74b923b6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/3ae08c1fe5f74f73b55edbef74b923b6 2024-12-09T06:17:37,090 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/3ae08c1fe5f74f73b55edbef74b923b6, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T06:17:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-09T06:17:37,090 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c69d9e9023ef26767a3f09a43f2c05f1 in 927ms, sequenceid=100, compaction requested=false 2024-12-09T06:17:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:37,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-09T06:17:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-09T06:17:37,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T06:17:37,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6970 sec 2024-12-09T06:17:37,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.7050 sec 2024-12-09T06:17:37,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T06:17:37,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:37,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:37,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:37,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:37,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:37,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:37,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/29d4c716e5564033b36b4372ba616b27 is 50, key is test_row_0/A:col10/1733725056459/Put/seqid=0 2024-12-09T06:17:37,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742277_1453 (size=14341) 2024-12-09T06:17:37,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/29d4c716e5564033b36b4372ba616b27 2024-12-09T06:17:37,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/08618ad5f7e7489bb2e2dcf3f00a1667 is 50, key is test_row_0/B:col10/1733725056459/Put/seqid=0 2024-12-09T06:17:37,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742278_1454 (size=12001) 2024-12-09T06:17:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725117140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725117142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725117143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725117266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725117266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725117266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725117477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725117477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725117477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725117496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725117496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,501 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in c69d9e9023ef26767a3f09a43f2c05f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,503 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in c69d9e9023ef26767a3f09a43f2c05f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/08618ad5f7e7489bb2e2dcf3f00a1667 2024-12-09T06:17:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T06:17:37,537 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:37,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/04c28fd89760416d878152175408a12d is 50, key is test_row_0/C:col10/1733725056459/Put/seqid=0 2024-12-09T06:17:37,539 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-09T06:17:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T06:17:37,542 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:37,542 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:37,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:37,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742279_1455 (size=12001) 2024-12-09T06:17:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T06:17:37,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T06:17:37,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:37,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:37,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:37,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:37,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:37,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725117787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725117787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725117796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:37,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T06:17:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:37,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:37,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T06:17:37,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/04c28fd89760416d878152175408a12d 2024-12-09T06:17:37,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/29d4c716e5564033b36b4372ba616b27 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/29d4c716e5564033b36b4372ba616b27 2024-12-09T06:17:37,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/29d4c716e5564033b36b4372ba616b27, entries=200, sequenceid=119, filesize=14.0 K 2024-12-09T06:17:37,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/08618ad5f7e7489bb2e2dcf3f00a1667 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/08618ad5f7e7489bb2e2dcf3f00a1667 2024-12-09T06:17:37,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/08618ad5f7e7489bb2e2dcf3f00a1667, entries=150, sequenceid=119, filesize=11.7 K 2024-12-09T06:17:37,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/04c28fd89760416d878152175408a12d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/04c28fd89760416d878152175408a12d 2024-12-09T06:17:37,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/04c28fd89760416d878152175408a12d, entries=150, sequenceid=119, filesize=11.7 K 2024-12-09T06:17:37,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c69d9e9023ef26767a3f09a43f2c05f1 in 882ms, sequenceid=119, compaction requested=true 2024-12-09T06:17:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:37,983 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:37,983 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:37,983 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:37,983 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:37,983 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:37,983 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:37,984 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:37,984 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:37,984 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f02fec71447b4c58aa8cf392a5af885a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/64f5470ac28446c889ed1608502d1a7e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/08618ad5f7e7489bb2e2dcf3f00a1667] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.3 K 2024-12-09T06:17:37,984 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ff63515bb8d44afb929f5e7479be0d31, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0f0fb84c8ea24a91ae501d01497e30ad, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/29d4c716e5564033b36b4372ba616b27] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=37.5 K 2024-12-09T06:17:37,984 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff63515bb8d44afb929f5e7479be0d31, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733725054127 2024-12-09T06:17:37,984 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f02fec71447b4c58aa8cf392a5af885a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733725054127 2024-12-09T06:17:37,984 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f0fb84c8ea24a91ae501d01497e30ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733725055302 2024-12-09T06:17:37,984 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 64f5470ac28446c889ed1608502d1a7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733725055302 2024-12-09T06:17:37,984 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d4c716e5564033b36b4372ba616b27, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733725056459 2024-12-09T06:17:37,984 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 08618ad5f7e7489bb2e2dcf3f00a1667, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733725056459 2024-12-09T06:17:37,992 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:37,992 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/2ccf058c238c47239e6d3574b003c83d is 50, key is test_row_0/A:col10/1733725056459/Put/seqid=0 2024-12-09T06:17:37,993 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:37,994 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/e6a6c75312a449d296fa35faa765a0cb is 50, key is test_row_0/B:col10/1733725056459/Put/seqid=0 2024-12-09T06:17:38,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T06:17:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,006 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T06:17:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:38,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:38,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:38,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742281_1457 (size=12207) 2024-12-09T06:17:38,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9eedbf9df5c5498085b790c8d36941e8 is 50, key is test_row_0/A:col10/1733725057139/Put/seqid=0 2024-12-09T06:17:38,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742280_1456 (size=12207) 2024-12-09T06:17:38,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742282_1458 (size=12101) 2024-12-09T06:17:38,022 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/e6a6c75312a449d296fa35faa765a0cb as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6a6c75312a449d296fa35faa765a0cb 2024-12-09T06:17:38,022 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9eedbf9df5c5498085b790c8d36941e8 2024-12-09T06:17:38,025 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into e6a6c75312a449d296fa35faa765a0cb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:38,026 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:38,026 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725057983; duration=0sec 2024-12-09T06:17:38,026 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:38,026 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:38,026 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:38,026 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:38,026 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:38,026 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,027 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/d687b403bfa54417bcc01d1ccf66d252, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/3ae08c1fe5f74f73b55edbef74b923b6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/04c28fd89760416d878152175408a12d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.3 K 2024-12-09T06:17:38,027 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting d687b403bfa54417bcc01d1ccf66d252, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733725054127 2024-12-09T06:17:38,027 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae08c1fe5f74f73b55edbef74b923b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733725055302 2024-12-09T06:17:38,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/5e2c6db50eb04b28b811a66acc5a5e02 is 50, key is test_row_0/B:col10/1733725057139/Put/seqid=0 2024-12-09T06:17:38,027 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 04c28fd89760416d878152175408a12d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733725056459 2024-12-09T06:17:38,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742283_1459 (size=12101) 2024-12-09T06:17:38,033 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/5e2c6db50eb04b28b811a66acc5a5e02 2024-12-09T06:17:38,036 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:38,036 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/21f4589822b344c19d36d674a6d304da is 50, key is test_row_0/C:col10/1733725056459/Put/seqid=0 2024-12-09T06:17:38,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/533a018bad9e41489a59f31a4ea8fc65 is 50, key is test_row_0/C:col10/1733725057139/Put/seqid=0 2024-12-09T06:17:38,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742285_1461 (size=12101) 2024-12-09T06:17:38,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742284_1460 (size=12207) 2024-12-09T06:17:38,046 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/533a018bad9e41489a59f31a4ea8fc65 2024-12-09T06:17:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9eedbf9df5c5498085b790c8d36941e8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9eedbf9df5c5498085b790c8d36941e8 2024-12-09T06:17:38,053 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/21f4589822b344c19d36d674a6d304da as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/21f4589822b344c19d36d674a6d304da 2024-12-09T06:17:38,058 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9eedbf9df5c5498085b790c8d36941e8, entries=150, sequenceid=135, filesize=11.8 K 2024-12-09T06:17:38,058 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into 21f4589822b344c19d36d674a6d304da(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:38,058 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:38,059 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725057983; duration=0sec 2024-12-09T06:17:38,059 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:38,059 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:38,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/5e2c6db50eb04b28b811a66acc5a5e02 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e2c6db50eb04b28b811a66acc5a5e02 2024-12-09T06:17:38,062 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e2c6db50eb04b28b811a66acc5a5e02, entries=150, sequenceid=135, filesize=11.8 K 2024-12-09T06:17:38,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/533a018bad9e41489a59f31a4ea8fc65 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/533a018bad9e41489a59f31a4ea8fc65 2024-12-09T06:17:38,066 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/533a018bad9e41489a59f31a4ea8fc65, entries=150, sequenceid=135, filesize=11.8 K 2024-12-09T06:17:38,066 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for c69d9e9023ef26767a3f09a43f2c05f1 in 60ms, sequenceid=135, compaction requested=false 2024-12-09T06:17:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T06:17:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-09T06:17:38,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T06:17:38,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 526 msec 2024-12-09T06:17:38,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 531 msec 2024-12-09T06:17:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T06:17:38,167 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:38,167 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-09T06:17:38,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T06:17:38,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:38,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T06:17:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:38,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:17:38,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:38,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:38,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:38,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:38,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:38,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:38,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:38,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/3987e2c645644937b47b631b75db3ed4 is 50, key is test_row_0/A:col10/1733725058313/Put/seqid=0 2024-12-09T06:17:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742286_1462 (size=14541) 2024-12-09T06:17:38,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725058344, max timestamp=1733725058367Num uniques 859; 2024-12-09T06:17:38,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725058344, max timestamp=1733725058367Num uniques 859; 2024-12-09T06:17:38,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:38,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:38,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725058344, max timestamp=1733725058367Num uniques 859; 2024-12-09T06:17:38,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:38,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725118403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725118407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725118412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,425 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/2ccf058c238c47239e6d3574b003c83d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/2ccf058c238c47239e6d3574b003c83d 2024-12-09T06:17:38,428 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into 2ccf058c238c47239e6d3574b003c83d(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:38,428 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:38,429 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725057983; duration=0sec 2024-12-09T06:17:38,429 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:38,429 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:38,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:38,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:38,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T06:17:38,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725118517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725118517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725118526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:38,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:38,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/3987e2c645644937b47b631b75db3ed4 2024-12-09T06:17:38,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/1103a4b4e93b408d8ee03ccc46342a19 is 50, key is test_row_0/B:col10/1733725058313/Put/seqid=0 2024-12-09T06:17:38,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725118727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725118727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742287_1463 (size=12151) 2024-12-09T06:17:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725118736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:38,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T06:17:38,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:38,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725119046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725119046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725119047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:39,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:39,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/1103a4b4e93b408d8ee03ccc46342a19 2024-12-09T06:17:39,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/30ef95ab4fec4f9b8d7d056d10073d8d is 50, key is test_row_0/C:col10/1733725058313/Put/seqid=0 2024-12-09T06:17:39,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742288_1464 (size=12151) 2024-12-09T06:17:39,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T06:17:39,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:39,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:39,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725119506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:39,515 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in c69d9e9023ef26767a3f09a43f2c05f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:39,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725119527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:39,538 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in c69d9e9023ef26767a3f09a43f2c05f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:39,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:39,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:39,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/30ef95ab4fec4f9b8d7d056d10073d8d 2024-12-09T06:17:39,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/3987e2c645644937b47b631b75db3ed4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/3987e2c645644937b47b631b75db3ed4 2024-12-09T06:17:39,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/3987e2c645644937b47b631b75db3ed4, entries=200, sequenceid=147, filesize=14.2 K 2024-12-09T06:17:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/1103a4b4e93b408d8ee03ccc46342a19 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1103a4b4e93b408d8ee03ccc46342a19 2024-12-09T06:17:39,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1103a4b4e93b408d8ee03ccc46342a19, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T06:17:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/30ef95ab4fec4f9b8d7d056d10073d8d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/30ef95ab4fec4f9b8d7d056d10073d8d 2024-12-09T06:17:39,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725119567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725119567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725119567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:39,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/30ef95ab4fec4f9b8d7d056d10073d8d, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T06:17:39,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c69d9e9023ef26767a3f09a43f2c05f1 in 1266ms, sequenceid=147, compaction requested=true 2024-12-09T06:17:39,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:39,584 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:39,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:39,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:39,584 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:39,585 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38849 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:39,585 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:39,585 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,585 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/2ccf058c238c47239e6d3574b003c83d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9eedbf9df5c5498085b790c8d36941e8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/3987e2c645644937b47b631b75db3ed4] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=37.9 K 2024-12-09T06:17:39,585 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ccf058c238c47239e6d3574b003c83d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733725056459 2024-12-09T06:17:39,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:39,585 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:39,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:39,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:39,585 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:39,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:39,585 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,586 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6a6c75312a449d296fa35faa765a0cb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e2c6db50eb04b28b811a66acc5a5e02, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1103a4b4e93b408d8ee03ccc46342a19] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.6 K 2024-12-09T06:17:39,586 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a6c75312a449d296fa35faa765a0cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733725056459 2024-12-09T06:17:39,590 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eedbf9df5c5498085b790c8d36941e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733725057117 2024-12-09T06:17:39,590 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e2c6db50eb04b28b811a66acc5a5e02, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733725057117 2024-12-09T06:17:39,591 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 1103a4b4e93b408d8ee03ccc46342a19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733725058313 2024-12-09T06:17:39,591 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3987e2c645644937b47b631b75db3ed4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733725058311 2024-12-09T06:17:39,612 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:39,613 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:39,613 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9599a0a55b88420b9f57a0cdfd328e18 is 50, key is test_row_0/A:col10/1733725058313/Put/seqid=0 2024-12-09T06:17:39,613 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/8682d2429d02404db28f12012ec8e919 is 50, key is test_row_0/B:col10/1733725058313/Put/seqid=0 2024-12-09T06:17:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742290_1466 (size=12459) 2024-12-09T06:17:39,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742289_1465 (size=12459) 2024-12-09T06:17:39,691 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/8682d2429d02404db28f12012ec8e919 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8682d2429d02404db28f12012ec8e919 2024-12-09T06:17:39,694 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into 8682d2429d02404db28f12012ec8e919(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:39,694 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:39,694 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725059584; duration=0sec 2024-12-09T06:17:39,695 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:39,695 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:39,695 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:39,696 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:39,696 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:39,696 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,696 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/21f4589822b344c19d36d674a6d304da, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/533a018bad9e41489a59f31a4ea8fc65, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/30ef95ab4fec4f9b8d7d056d10073d8d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.6 K 2024-12-09T06:17:39,696 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 21f4589822b344c19d36d674a6d304da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733725056459 2024-12-09T06:17:39,697 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 533a018bad9e41489a59f31a4ea8fc65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733725057117 2024-12-09T06:17:39,697 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ef95ab4fec4f9b8d7d056d10073d8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733725058313 2024-12-09T06:17:39,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T06:17:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:39,698 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-09T06:17:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/702bc97c62fe46b983fe0a132fceae2f is 50, key is test_row_0/A:col10/1733725058405/Put/seqid=0 2024-12-09T06:17:39,715 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:39,716 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/b431b91b2a5b4b4f95df6f3a84b5463a is 50, key is test_row_0/C:col10/1733725058313/Put/seqid=0 2024-12-09T06:17:39,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742291_1467 (size=12151) 2024-12-09T06:17:39,718 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/702bc97c62fe46b983fe0a132fceae2f 2024-12-09T06:17:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/5b1de720f601475aae10cc3972caf02f is 50, key is test_row_0/B:col10/1733725058405/Put/seqid=0 2024-12-09T06:17:39,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742292_1468 (size=12459) 2024-12-09T06:17:39,738 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/b431b91b2a5b4b4f95df6f3a84b5463a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b431b91b2a5b4b4f95df6f3a84b5463a 2024-12-09T06:17:39,744 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into b431b91b2a5b4b4f95df6f3a84b5463a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:39,744 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:39,744 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725059585; duration=0sec 2024-12-09T06:17:39,744 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:39,744 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:39,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742293_1469 (size=12151) 2024-12-09T06:17:39,881 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T06:17:39,881 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T06:17:40,069 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9599a0a55b88420b9f57a0cdfd328e18 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9599a0a55b88420b9f57a0cdfd328e18 2024-12-09T06:17:40,077 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into 9599a0a55b88420b9f57a0cdfd328e18(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:40,077 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:40,077 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725059584; duration=0sec 2024-12-09T06:17:40,077 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:40,077 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:40,150 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/5b1de720f601475aae10cc3972caf02f 2024-12-09T06:17:40,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/03bac667e12d4c429b4f83a179cc241a is 50, key is test_row_0/C:col10/1733725058405/Put/seqid=0 2024-12-09T06:17:40,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742294_1470 (size=12151) 2024-12-09T06:17:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T06:17:40,577 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/03bac667e12d4c429b4f83a179cc241a 2024-12-09T06:17:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/702bc97c62fe46b983fe0a132fceae2f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/702bc97c62fe46b983fe0a132fceae2f 2024-12-09T06:17:40,597 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/702bc97c62fe46b983fe0a132fceae2f, entries=150, sequenceid=184, filesize=11.9 K 2024-12-09T06:17:40,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/5b1de720f601475aae10cc3972caf02f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5b1de720f601475aae10cc3972caf02f 2024-12-09T06:17:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:40,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:40,605 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5b1de720f601475aae10cc3972caf02f, entries=150, sequenceid=184, filesize=11.9 K 2024-12-09T06:17:40,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-09T06:17:40,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/03bac667e12d4c429b4f83a179cc241a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/03bac667e12d4c429b4f83a179cc241a 2024-12-09T06:17:40,612 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/03bac667e12d4c429b4f83a179cc241a, entries=150, sequenceid=184, filesize=11.9 K 2024-12-09T06:17:40,613 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=33.54 KB/34350 for c69d9e9023ef26767a3f09a43f2c05f1 in 915ms, sequenceid=184, compaction requested=false 2024-12-09T06:17:40,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:40,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:40,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T06:17:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-09T06:17:40,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T06:17:40,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4460 sec 2024-12-09T06:17:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:40,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:17:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:40,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.4510 sec 2024-12-09T06:17:40,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/38995b8927184bdab370ae63cde7a8dc is 50, key is test_row_0/A:col10/1733725060617/Put/seqid=0 2024-12-09T06:17:40,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742295_1471 (size=12151) 2024-12-09T06:17:40,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/38995b8927184bdab370ae63cde7a8dc 2024-12-09T06:17:40,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/fbf5b20fdbc142688c1f015a6f8ef54c is 50, key is test_row_0/B:col10/1733725060617/Put/seqid=0 2024-12-09T06:17:40,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742296_1472 (size=12151) 2024-12-09T06:17:40,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/fbf5b20fdbc142688c1f015a6f8ef54c 2024-12-09T06:17:40,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/be08b86300024f99b80b502ffb117065 is 50, key is test_row_0/C:col10/1733725060617/Put/seqid=0 2024-12-09T06:17:40,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725060647, max timestamp=1733725060692Num uniques 859; 2024-12-09T06:17:40,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725060647, max timestamp=1733725060692Num uniques 859; 2024-12-09T06:17:40,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:40,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:40,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725060647, max timestamp=1733725060692Num uniques 859; 2024-12-09T06:17:40,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:40,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742297_1473 (size=12151) 2024-12-09T06:17:40,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/be08b86300024f99b80b502ffb117065 2024-12-09T06:17:40,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/38995b8927184bdab370ae63cde7a8dc as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/38995b8927184bdab370ae63cde7a8dc 2024-12-09T06:17:40,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/38995b8927184bdab370ae63cde7a8dc, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T06:17:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/fbf5b20fdbc142688c1f015a6f8ef54c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fbf5b20fdbc142688c1f015a6f8ef54c 2024-12-09T06:17:40,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fbf5b20fdbc142688c1f015a6f8ef54c, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T06:17:40,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/be08b86300024f99b80b502ffb117065 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/be08b86300024f99b80b502ffb117065 2024-12-09T06:17:40,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/be08b86300024f99b80b502ffb117065, entries=150, sequenceid=197, filesize=11.9 K 2024-12-09T06:17:40,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=93.93 KB/96180 for c69d9e9023ef26767a3f09a43f2c05f1 in 108ms, sequenceid=197, compaction requested=true 2024-12-09T06:17:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:40,727 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:40,727 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:40,728 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:40,728 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:40,728 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:40,728 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:40,728 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:40,728 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:40,728 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9599a0a55b88420b9f57a0cdfd328e18, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/702bc97c62fe46b983fe0a132fceae2f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/38995b8927184bdab370ae63cde7a8dc] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.9 K 2024-12-09T06:17:40,728 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8682d2429d02404db28f12012ec8e919, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5b1de720f601475aae10cc3972caf02f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fbf5b20fdbc142688c1f015a6f8ef54c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.9 K 2024-12-09T06:17:40,729 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9599a0a55b88420b9f57a0cdfd328e18, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733725058313 2024-12-09T06:17:40,729 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8682d2429d02404db28f12012ec8e919, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733725058313 2024-12-09T06:17:40,729 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1de720f601475aae10cc3972caf02f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733725058402 2024-12-09T06:17:40,729 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702bc97c62fe46b983fe0a132fceae2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733725058402 2024-12-09T06:17:40,729 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38995b8927184bdab370ae63cde7a8dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733725060610 2024-12-09T06:17:40,729 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf5b20fdbc142688c1f015a6f8ef54c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733725060610 2024-12-09T06:17:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:40,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=107.34 KB heapSize=295.92 KB 2024-12-09T06:17:40,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:40,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:40,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:40,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:40,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:40,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:40,747 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:40,748 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/f0f781b869ce48ab895961676be2ccda is 50, key is test_row_0/A:col10/1733725060617/Put/seqid=0 2024-12-09T06:17:40,756 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:40,756 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/8eefe9db47604759b8e486a808898124 is 50, key is test_row_0/B:col10/1733725060617/Put/seqid=0 2024-12-09T06:17:40,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/1d129a5ea7074c4cbc7e11ae9aedb01f is 50, key is test_row_0/A:col10/1733725060738/Put/seqid=0 2024-12-09T06:17:40,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742299_1475 (size=12561) 2024-12-09T06:17:40,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725120797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725120798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725120801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:40,829 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/8eefe9db47604759b8e486a808898124 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8eefe9db47604759b8e486a808898124 2024-12-09T06:17:40,834 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into 8eefe9db47604759b8e486a808898124(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:40,834 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:40,834 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725060727; duration=0sec 2024-12-09T06:17:40,834 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:40,835 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:40,835 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742298_1474 (size=12561) 2024-12-09T06:17:40,836 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:40,836 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:40,836 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:40,836 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b431b91b2a5b4b4f95df6f3a84b5463a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/03bac667e12d4c429b4f83a179cc241a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/be08b86300024f99b80b502ffb117065] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=35.9 K 2024-12-09T06:17:40,836 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b431b91b2a5b4b4f95df6f3a84b5463a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733725058313 2024-12-09T06:17:40,837 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 03bac667e12d4c429b4f83a179cc241a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1733725058402 2024-12-09T06:17:40,837 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting be08b86300024f99b80b502ffb117065, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733725060610 2024-12-09T06:17:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742300_1476 (size=19321) 2024-12-09T06:17:40,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/1d129a5ea7074c4cbc7e11ae9aedb01f 2024-12-09T06:17:40,866 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:40,866 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/de57e72c6bb04f22a1565a9082555bd3 is 50, key is test_row_0/C:col10/1733725060617/Put/seqid=0 2024-12-09T06:17:40,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/2b32f2660d684f029948a78d95c7219b is 50, key is test_row_0/B:col10/1733725060738/Put/seqid=0 2024-12-09T06:17:40,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742301_1477 (size=12561) 2024-12-09T06:17:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742302_1478 (size=12151) 2024-12-09T06:17:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725120927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:40,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725120927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:40,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725120927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:41,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725121137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:41,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725121147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725121147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:41,238 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/f0f781b869ce48ab895961676be2ccda as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/f0f781b869ce48ab895961676be2ccda 2024-12-09T06:17:41,243 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into f0f781b869ce48ab895961676be2ccda(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:41,243 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:41,243 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725060727; duration=0sec 2024-12-09T06:17:41,244 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:41,244 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:41,299 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/de57e72c6bb04f22a1565a9082555bd3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/de57e72c6bb04f22a1565a9082555bd3 2024-12-09T06:17:41,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/2b32f2660d684f029948a78d95c7219b 2024-12-09T06:17:41,304 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into de57e72c6bb04f22a1565a9082555bd3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:41,304 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:41,304 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725060727; duration=0sec 2024-12-09T06:17:41,304 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:41,304 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:41,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/848154c843074f2dba8f73215f409f0f is 50, key is test_row_0/C:col10/1733725060738/Put/seqid=0 2024-12-09T06:17:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742303_1479 (size=12151) 2024-12-09T06:17:41,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725121446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:41,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725121456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:41,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725121467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:41,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/848154c843074f2dba8f73215f409f0f 2024-12-09T06:17:41,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/1d129a5ea7074c4cbc7e11ae9aedb01f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/1d129a5ea7074c4cbc7e11ae9aedb01f 2024-12-09T06:17:41,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/1d129a5ea7074c4cbc7e11ae9aedb01f, entries=300, sequenceid=229, filesize=18.9 K 2024-12-09T06:17:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/2b32f2660d684f029948a78d95c7219b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/2b32f2660d684f029948a78d95c7219b 2024-12-09T06:17:41,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/2b32f2660d684f029948a78d95c7219b, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T06:17:41,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/848154c843074f2dba8f73215f409f0f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/848154c843074f2dba8f73215f409f0f 2024-12-09T06:17:41,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/848154c843074f2dba8f73215f409f0f, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T06:17:41,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~315.80 KB/323376, currentSize=67.09 KB/68700 for c69d9e9023ef26767a3f09a43f2c05f1 in 992ms, sequenceid=229, compaction requested=false 2024-12-09T06:17:41,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:41,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T06:17:41,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:41,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:41,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:41,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:41,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:41,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:41,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/90abd74af2774f349910c851ade24a85 is 50, key is test_row_0/A:col10/1733725060796/Put/seqid=0 2024-12-09T06:17:41,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742304_1480 (size=14541) 2024-12-09T06:17:42,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725122003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725122009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725122010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725122126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725122127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725122127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T06:17:42,337 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:42,337 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-09T06:17:42,339 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T06:17:42,340 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:42,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:42,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725122346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725122347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725122347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/90abd74af2774f349910c851ade24a85 2024-12-09T06:17:42,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/10eecd2142b54442a3c2ced9c551e53c is 50, key is test_row_0/B:col10/1733725060796/Put/seqid=0 2024-12-09T06:17:42,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742305_1481 (size=12151) 2024-12-09T06:17:42,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/10eecd2142b54442a3c2ced9c551e53c 2024-12-09T06:17:42,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/c74e9a47dff5427c9761683380053967 is 50, key is test_row_0/C:col10/1733725060796/Put/seqid=0 2024-12-09T06:17:42,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742306_1482 (size=12151) 2024-12-09T06:17:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T06:17:42,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T06:17:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T06:17:42,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:42,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T06:17:42,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725122667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725122667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725122667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:42,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T06:17:42,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:42,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:42,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/c74e9a47dff5427c9761683380053967 2024-12-09T06:17:42,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/90abd74af2774f349910c851ade24a85 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/90abd74af2774f349910c851ade24a85 2024-12-09T06:17:42,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/90abd74af2774f349910c851ade24a85, entries=200, sequenceid=246, filesize=14.2 K 2024-12-09T06:17:42,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/10eecd2142b54442a3c2ced9c551e53c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/10eecd2142b54442a3c2ced9c551e53c 2024-12-09T06:17:42,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/10eecd2142b54442a3c2ced9c551e53c, entries=150, sequenceid=246, filesize=11.9 K 2024-12-09T06:17:42,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/c74e9a47dff5427c9761683380053967 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/c74e9a47dff5427c9761683380053967 2024-12-09T06:17:42,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/c74e9a47dff5427c9761683380053967, entries=150, sequenceid=246, filesize=11.9 K 2024-12-09T06:17:42,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c69d9e9023ef26767a3f09a43f2c05f1 in 891ms, sequenceid=246, compaction requested=true 2024-12-09T06:17:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:42,850 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:42,850 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:42,851 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:42,851 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 46423 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:42,851 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:42,851 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:42,851 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,851 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,851 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8eefe9db47604759b8e486a808898124, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/2b32f2660d684f029948a78d95c7219b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/10eecd2142b54442a3c2ced9c551e53c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=36.0 K 2024-12-09T06:17:42,851 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/f0f781b869ce48ab895961676be2ccda, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/1d129a5ea7074c4cbc7e11ae9aedb01f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/90abd74af2774f349910c851ade24a85] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=45.3 K 2024-12-09T06:17:42,851 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eefe9db47604759b8e486a808898124, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733725060610 2024-12-09T06:17:42,851 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f781b869ce48ab895961676be2ccda, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733725060610 2024-12-09T06:17:42,852 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b32f2660d684f029948a78d95c7219b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733725060723 2024-12-09T06:17:42,852 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d129a5ea7074c4cbc7e11ae9aedb01f, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733725060720 2024-12-09T06:17:42,852 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 10eecd2142b54442a3c2ced9c551e53c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733725060796 2024-12-09T06:17:42,852 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90abd74af2774f349910c851ade24a85, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733725060794 2024-12-09T06:17:42,856 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:42,857 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/de0d68c6dcc74729872d1c5823cdeef7 is 50, key is test_row_0/B:col10/1733725060796/Put/seqid=0 2024-12-09T06:17:42,862 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:42,862 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9316d49799aa490488994f06fa4a54a0 is 50, key is test_row_0/A:col10/1733725060796/Put/seqid=0 2024-12-09T06:17:42,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742307_1483 (size=12663) 2024-12-09T06:17:42,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742308_1484 (size=12663) 2024-12-09T06:17:42,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T06:17:42,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:42,962 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T06:17:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:42,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:42,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/49aa615c8b0d4173aca5a48cf215465b is 50, key is test_row_0/A:col10/1733725062002/Put/seqid=0 2024-12-09T06:17:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T06:17:42,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742309_1485 (size=12301) 2024-12-09T06:17:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:43,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:43,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725123215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725123215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725123217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,276 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/de0d68c6dcc74729872d1c5823cdeef7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/de0d68c6dcc74729872d1c5823cdeef7 2024-12-09T06:17:43,278 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9316d49799aa490488994f06fa4a54a0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9316d49799aa490488994f06fa4a54a0 2024-12-09T06:17:43,297 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into de0d68c6dcc74729872d1c5823cdeef7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:43,297 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:43,297 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725062850; duration=0sec 2024-12-09T06:17:43,297 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:43,297 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:43,297 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:43,302 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:43,302 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:43,302 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:43,302 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/de57e72c6bb04f22a1565a9082555bd3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/848154c843074f2dba8f73215f409f0f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/c74e9a47dff5427c9761683380053967] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=36.0 K 2024-12-09T06:17:43,303 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting de57e72c6bb04f22a1565a9082555bd3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733725060610 2024-12-09T06:17:43,303 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 848154c843074f2dba8f73215f409f0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733725060723 2024-12-09T06:17:43,303 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into 9316d49799aa490488994f06fa4a54a0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:43,303 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:43,303 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725062850; duration=0sec 2024-12-09T06:17:43,303 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:43,303 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:43,304 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c74e9a47dff5427c9761683380053967, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733725060796 2024-12-09T06:17:43,315 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:43,315 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/17777a8afbc0461baa7601ee3f1865e1 is 50, key is test_row_0/C:col10/1733725060796/Put/seqid=0 2024-12-09T06:17:43,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742310_1486 (size=12663) 2024-12-09T06:17:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725123336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725123336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725123337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,368 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/49aa615c8b0d4173aca5a48cf215465b 2024-12-09T06:17:43,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/02ab8614e39f48768313e2dd612abc38 is 50, key is test_row_0/B:col10/1733725062002/Put/seqid=0 2024-12-09T06:17:43,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742311_1487 (size=12301) 2024-12-09T06:17:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T06:17:43,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725123547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725123547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59416 deadline: 1733725123547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725123547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,554 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in c69d9e9023ef26767a3f09a43f2c05f1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59442 deadline: 1733725123566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,575 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_0' in c69d9e9023ef26767a3f09a43f2c05f1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,724 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/17777a8afbc0461baa7601ee3f1865e1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17777a8afbc0461baa7601ee3f1865e1 2024-12-09T06:17:43,729 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into 17777a8afbc0461baa7601ee3f1865e1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:43,729 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:43,729 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725062850; duration=0sec 2024-12-09T06:17:43,729 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:43,729 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:43,803 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/02ab8614e39f48768313e2dd612abc38 2024-12-09T06:17:43,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/518d954b359f492783a00015ae1d4943 is 50, key is test_row_0/C:col10/1733725062002/Put/seqid=0 2024-12-09T06:17:43,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742312_1488 (size=12301) 2024-12-09T06:17:43,839 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/518d954b359f492783a00015ae1d4943 2024-12-09T06:17:43,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/49aa615c8b0d4173aca5a48cf215465b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/49aa615c8b0d4173aca5a48cf215465b 2024-12-09T06:17:43,869 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/49aa615c8b0d4173aca5a48cf215465b, entries=150, sequenceid=264, filesize=12.0 K 2024-12-09T06:17:43,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725123857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725123857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/02ab8614e39f48768313e2dd612abc38 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/02ab8614e39f48768313e2dd612abc38 2024-12-09T06:17:43,875 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/02ab8614e39f48768313e2dd612abc38, entries=150, sequenceid=264, filesize=12.0 K 2024-12-09T06:17:43,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/518d954b359f492783a00015ae1d4943 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/518d954b359f492783a00015ae1d4943 2024-12-09T06:17:43,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725123867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:43,881 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/518d954b359f492783a00015ae1d4943, entries=150, sequenceid=264, filesize=12.0 K 2024-12-09T06:17:43,883 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c69d9e9023ef26767a3f09a43f2c05f1 in 920ms, sequenceid=264, compaction requested=false 2024-12-09T06:17:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:43,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-09T06:17:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-09T06:17:43,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-09T06:17:43,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-12-09T06:17:43,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5510 sec 2024-12-09T06:17:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:44,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T06:17:44,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:44,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:44,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:44,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:44,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:44,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:44,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/8ab0ffac6407461a86caa969af4473d0 is 50, key is test_row_0/A:col10/1733725063215/Put/seqid=0 2024-12-09T06:17:44,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742313_1489 (size=12301) 2024-12-09T06:17:44,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/8ab0ffac6407461a86caa969af4473d0 2024-12-09T06:17:44,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/0ba9f7217e9a47fd94f2c6f9d0993f26 is 50, key is test_row_0/B:col10/1733725063215/Put/seqid=0 2024-12-09T06:17:44,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725124421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725124433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725124434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742314_1490 (size=12301) 2024-12-09T06:17:44,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/0ba9f7217e9a47fd94f2c6f9d0993f26 2024-12-09T06:17:44,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/5c7aeebfb7644d23a7630d2a879b7d2e is 50, key is test_row_0/C:col10/1733725063215/Put/seqid=0 2024-12-09T06:17:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T06:17:44,487 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:44,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742315_1491 (size=12301) 2024-12-09T06:17:44,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:44,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/5c7aeebfb7644d23a7630d2a879b7d2e 2024-12-09T06:17:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-09T06:17:44,491 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:44,494 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:44,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T06:17:44,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/8ab0ffac6407461a86caa969af4473d0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8ab0ffac6407461a86caa969af4473d0 2024-12-09T06:17:44,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8ab0ffac6407461a86caa969af4473d0, entries=150, sequenceid=282, filesize=12.0 K 2024-12-09T06:17:44,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/0ba9f7217e9a47fd94f2c6f9d0993f26 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/0ba9f7217e9a47fd94f2c6f9d0993f26 2024-12-09T06:17:44,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/0ba9f7217e9a47fd94f2c6f9d0993f26, entries=150, sequenceid=282, filesize=12.0 K 2024-12-09T06:17:44,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/5c7aeebfb7644d23a7630d2a879b7d2e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5c7aeebfb7644d23a7630d2a879b7d2e 2024-12-09T06:17:44,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5c7aeebfb7644d23a7630d2a879b7d2e, entries=150, sequenceid=282, filesize=12.0 K 2024-12-09T06:17:44,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c69d9e9023ef26767a3f09a43f2c05f1 in 144ms, sequenceid=282, compaction requested=true 2024-12-09T06:17:44,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:44,525 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:44,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:44,525 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:44,527 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:44,527 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:44,527 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:44,527 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:44,527 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,527 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,527 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/de0d68c6dcc74729872d1c5823cdeef7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/02ab8614e39f48768313e2dd612abc38, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/0ba9f7217e9a47fd94f2c6f9d0993f26] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=36.4 K 2024-12-09T06:17:44,527 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9316d49799aa490488994f06fa4a54a0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/49aa615c8b0d4173aca5a48cf215465b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8ab0ffac6407461a86caa969af4473d0] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=36.4 K 2024-12-09T06:17:44,527 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting de0d68c6dcc74729872d1c5823cdeef7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733725060796 2024-12-09T06:17:44,527 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9316d49799aa490488994f06fa4a54a0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733725060796 2024-12-09T06:17:44,529 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49aa615c8b0d4173aca5a48cf215465b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733725062002 2024-12-09T06:17:44,529 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab0ffac6407461a86caa969af4473d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733725063213 2024-12-09T06:17:44,529 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ab8614e39f48768313e2dd612abc38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733725062002 2024-12-09T06:17:44,530 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ba9f7217e9a47fd94f2c6f9d0993f26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733725063213 2024-12-09T06:17:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:44,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T06:17:44,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:44,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:44,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:44,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:44,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:44,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:44,547 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:44,547 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/5550c140033a46798d0f306c7f536e87 is 50, key is test_row_0/A:col10/1733725063215/Put/seqid=0 2024-12-09T06:17:44,550 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:44,550 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/c27f31ad7af64be394848a17fa5436a1 is 50, key is test_row_0/B:col10/1733725063215/Put/seqid=0 2024-12-09T06:17:44,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/913038cd1e624082b3611e11865e7c37 is 50, key is test_row_0/A:col10/1733725064537/Put/seqid=0 2024-12-09T06:17:44,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742316_1492 (size=12915) 2024-12-09T06:17:44,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742317_1493 (size=12915) 2024-12-09T06:17:44,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742318_1494 (size=14741) 2024-12-09T06:17:44,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725124580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725124581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725124582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T06:17:44,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:44,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:44,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725124696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725124696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725124696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:44,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T06:17:44,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725124906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725124909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725124909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:44,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:44,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:44,969 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/5550c140033a46798d0f306c7f536e87 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5550c140033a46798d0f306c7f536e87 2024-12-09T06:17:44,973 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into 5550c140033a46798d0f306c7f536e87(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:44,973 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:44,973 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725064524; duration=0sec 2024-12-09T06:17:44,973 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:44,973 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:44,973 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:44,974 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:44,974 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:44,974 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:44,974 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17777a8afbc0461baa7601ee3f1865e1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/518d954b359f492783a00015ae1d4943, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5c7aeebfb7644d23a7630d2a879b7d2e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=36.4 K 2024-12-09T06:17:44,974 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/c27f31ad7af64be394848a17fa5436a1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/c27f31ad7af64be394848a17fa5436a1 2024-12-09T06:17:44,974 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17777a8afbc0461baa7601ee3f1865e1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733725060796 2024-12-09T06:17:44,976 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 518d954b359f492783a00015ae1d4943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733725062002 2024-12-09T06:17:44,977 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c7aeebfb7644d23a7630d2a879b7d2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733725063213 2024-12-09T06:17:44,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/913038cd1e624082b3611e11865e7c37 2024-12-09T06:17:44,980 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into c27f31ad7af64be394848a17fa5436a1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:44,980 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:44,980 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725064525; duration=0sec 2024-12-09T06:17:44,980 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:44,980 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:44,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/e6f805ad7d3649a1b87f07b346f08a5c is 50, key is test_row_0/B:col10/1733725064537/Put/seqid=0 2024-12-09T06:17:44,986 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:44,986 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/9e85e61964a44baf88db4675f7a9ca5a is 50, key is test_row_0/C:col10/1733725063215/Put/seqid=0 2024-12-09T06:17:44,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742319_1495 (size=12301) 2024-12-09T06:17:45,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742320_1496 (size=12915) 2024-12-09T06:17:45,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T06:17:45,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725125227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:45,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725125227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:45,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725125227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:45,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:45,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:45,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/e6f805ad7d3649a1b87f07b346f08a5c 2024-12-09T06:17:45,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/585a37b9f61d49e192c50c9512b1460b is 50, key is test_row_0/C:col10/1733725064537/Put/seqid=0 2024-12-09T06:17:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742321_1497 (size=12301) 2024-12-09T06:17:45,413 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/9e85e61964a44baf88db4675f7a9ca5a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/9e85e61964a44baf88db4675f7a9ca5a 2024-12-09T06:17:45,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:45,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:45,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,417 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into 9e85e61964a44baf88db4675f7a9ca5a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:45,417 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:45,417 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725064525; duration=0sec 2024-12-09T06:17:45,417 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:45,417 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:45,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T06:17:45,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:45,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:45,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725125738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725125738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:45,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725125738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:45,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/585a37b9f61d49e192c50c9512b1460b 2024-12-09T06:17:45,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/913038cd1e624082b3611e11865e7c37 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/913038cd1e624082b3611e11865e7c37 2024-12-09T06:17:45,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/913038cd1e624082b3611e11865e7c37, entries=200, sequenceid=300, filesize=14.4 K 2024-12-09T06:17:45,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/e6f805ad7d3649a1b87f07b346f08a5c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6f805ad7d3649a1b87f07b346f08a5c 2024-12-09T06:17:45,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6f805ad7d3649a1b87f07b346f08a5c, entries=150, sequenceid=300, filesize=12.0 K 2024-12-09T06:17:45,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/585a37b9f61d49e192c50c9512b1460b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/585a37b9f61d49e192c50c9512b1460b 2024-12-09T06:17:45,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/585a37b9f61d49e192c50c9512b1460b, entries=150, sequenceid=300, filesize=12.0 K 2024-12-09T06:17:45,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c69d9e9023ef26767a3f09a43f2c05f1 in 1288ms, sequenceid=300, compaction requested=false 2024-12-09T06:17:45,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:45,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T06:17:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:45,873 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T06:17:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:45,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:45,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/50eb91d0f39b4b5da615c7480e52036f is 50, key is test_row_0/A:col10/1733725064581/Put/seqid=0 2024-12-09T06:17:45,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742322_1498 (size=12301) 2024-12-09T06:17:46,285 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/50eb91d0f39b4b5da615c7480e52036f 2024-12-09T06:17:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/fe9f7db0cb884f1aa0aa5f0143d958a3 is 50, key is test_row_0/B:col10/1733725064581/Put/seqid=0 2024-12-09T06:17:46,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742323_1499 (size=12301) 2024-12-09T06:17:46,336 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/fe9f7db0cb884f1aa0aa5f0143d958a3 2024-12-09T06:17:46,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/bfa61d07a0f841d9a77a04e039b8ab5b is 50, key is test_row_0/C:col10/1733725064581/Put/seqid=0 2024-12-09T06:17:46,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742324_1500 (size=12301) 2024-12-09T06:17:46,360 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/bfa61d07a0f841d9a77a04e039b8ab5b 2024-12-09T06:17:46,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/50eb91d0f39b4b5da615c7480e52036f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/50eb91d0f39b4b5da615c7480e52036f 2024-12-09T06:17:46,377 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/50eb91d0f39b4b5da615c7480e52036f, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T06:17:46,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/fe9f7db0cb884f1aa0aa5f0143d958a3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fe9f7db0cb884f1aa0aa5f0143d958a3 2024-12-09T06:17:46,400 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fe9f7db0cb884f1aa0aa5f0143d958a3, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T06:17:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/bfa61d07a0f841d9a77a04e039b8ab5b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/bfa61d07a0f841d9a77a04e039b8ab5b 2024-12-09T06:17:46,407 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/bfa61d07a0f841d9a77a04e039b8ab5b, entries=150, sequenceid=317, filesize=12.0 K 2024-12-09T06:17:46,408 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for c69d9e9023ef26767a3f09a43f2c05f1 in 535ms, sequenceid=317, compaction requested=true 2024-12-09T06:17:46,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:46,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:46,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-09T06:17:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-09T06:17:46,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T06:17:46,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9160 sec 2024-12-09T06:17:46,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.9240 sec 2024-12-09T06:17:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T06:17:46,647 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:46,647 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-09T06:17:46,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T06:17:46,650 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:46,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T06:17:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:46,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:17:46,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:46,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:46,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:46,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9953d389468b40a8bab0fc44ebe225d8 is 50, key is test_row_0/A:col10/1733725066784/Put/seqid=0 2024-12-09T06:17:46,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742325_1501 (size=14741) 2024-12-09T06:17:46,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9953d389468b40a8bab0fc44ebe225d8 2024-12-09T06:17:46,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/4e803dbc16094c6093fff921088c0a6a is 50, key is test_row_0/B:col10/1733725066784/Put/seqid=0 2024-12-09T06:17:46,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742326_1502 (size=12301) 2024-12-09T06:17:46,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/4e803dbc16094c6093fff921088c0a6a 2024-12-09T06:17:46,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/8ec13f8a84334944a5eae2990e3e8bb7 is 50, key is test_row_0/C:col10/1733725066784/Put/seqid=0 2024-12-09T06:17:46,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742327_1503 (size=12301) 2024-12-09T06:17:46,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733725066845, max timestamp=1733725066894Num uniques 859; 2024-12-09T06:17:46,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733725066845, max timestamp=1733725066894Num uniques 859; 2024-12-09T06:17:46,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:46,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:46,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/8ec13f8a84334944a5eae2990e3e8bb7 2024-12-09T06:17:46,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725066845, max timestamp=1733725066894Num uniques 859; 2024-12-09T06:17:46,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/9953d389468b40a8bab0fc44ebe225d8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9953d389468b40a8bab0fc44ebe225d8 2024-12-09T06:17:46,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9953d389468b40a8bab0fc44ebe225d8, entries=200, sequenceid=329, filesize=14.4 K 2024-12-09T06:17:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/4e803dbc16094c6093fff921088c0a6a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/4e803dbc16094c6093fff921088c0a6a 2024-12-09T06:17:46,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/4e803dbc16094c6093fff921088c0a6a, entries=150, sequenceid=329, filesize=12.0 K 2024-12-09T06:17:46,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/8ec13f8a84334944a5eae2990e3e8bb7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/8ec13f8a84334944a5eae2990e3e8bb7 2024-12-09T06:17:46,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/8ec13f8a84334944a5eae2990e3e8bb7, entries=150, sequenceid=329, filesize=12.0 K 2024-12-09T06:17:46,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=53.67 KB/54960 for c69d9e9023ef26767a3f09a43f2c05f1 in 135ms, sequenceid=329, compaction requested=true 2024-12-09T06:17:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:46,920 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:46,920 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:46,922 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:46,922 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:46,922 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:46,923 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:46,923 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:46,923 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:46,923 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5550c140033a46798d0f306c7f536e87, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/913038cd1e624082b3611e11865e7c37, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/50eb91d0f39b4b5da615c7480e52036f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9953d389468b40a8bab0fc44ebe225d8] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=53.4 K 2024-12-09T06:17:46,923 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/c27f31ad7af64be394848a17fa5436a1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6f805ad7d3649a1b87f07b346f08a5c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fe9f7db0cb884f1aa0aa5f0143d958a3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/4e803dbc16094c6093fff921088c0a6a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=48.7 K 2024-12-09T06:17:46,923 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5550c140033a46798d0f306c7f536e87, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733725063213 2024-12-09T06:17:46,924 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c27f31ad7af64be394848a17fa5436a1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733725063213 2024-12-09T06:17:46,924 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913038cd1e624082b3611e11865e7c37, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733725064419 2024-12-09T06:17:46,924 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f805ad7d3649a1b87f07b346f08a5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733725064419 2024-12-09T06:17:46,924 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50eb91d0f39b4b5da615c7480e52036f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733725064567 2024-12-09T06:17:46,924 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9f7db0cb884f1aa0aa5f0143d958a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733725064567 2024-12-09T06:17:46,925 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9953d389468b40a8bab0fc44ebe225d8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733725066773 2024-12-09T06:17:46,925 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e803dbc16094c6093fff921088c0a6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733725066773 2024-12-09T06:17:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:46,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=60.38 KB heapSize=161.16 KB 2024-12-09T06:17:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:46,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/abd61548f46441859e19eaa8881b06d9 is 50, key is test_row_0/A:col10/1733725066922/Put/seqid=0 2024-12-09T06:17:46,940 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:46,941 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/fe5623d8681d46fca2f9b47fc4a1338b is 50, key is test_row_0/A:col10/1733725066784/Put/seqid=0 2024-12-09T06:17:46,944 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:46,945 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/1ed1d6ab4dd94d1f97bdf058e4fb291e is 50, key is test_row_0/B:col10/1733725066784/Put/seqid=0 2024-12-09T06:17:46,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742328_1504 (size=14741) 2024-12-09T06:17:46,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742329_1505 (size=13051) 2024-12-09T06:17:46,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:46,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:46,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:46,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:46,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:46,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:46,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742330_1506 (size=13051) 2024-12-09T06:17:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T06:17:47,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725066979, max timestamp=1733725067009Num uniques 859; 2024-12-09T06:17:47,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733725066979, max timestamp=1733725067009Num uniques 859; 2024-12-09T06:17:47,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:47,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T06:17:47,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725066993, max timestamp=1733725067009Num uniques 859; 2024-12-09T06:17:47,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:47,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725127080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725127081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725127082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725127197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725127206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725127206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T06:17:47,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/abd61548f46441859e19eaa8881b06d9 2024-12-09T06:17:47,354 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/fe5623d8681d46fca2f9b47fc4a1338b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fe5623d8681d46fca2f9b47fc4a1338b 2024-12-09T06:17:47,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/f73169d6e7684ec1807b09ec0757f31b is 50, key is test_row_0/B:col10/1733725066922/Put/seqid=0 2024-12-09T06:17:47,357 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into fe5623d8681d46fca2f9b47fc4a1338b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:47,357 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:47,357 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=12, startTime=1733725066920; duration=0sec 2024-12-09T06:17:47,357 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:47,357 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:47,357 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:17:47,358 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:17:47,358 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:47,358 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,358 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/9e85e61964a44baf88db4675f7a9ca5a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/585a37b9f61d49e192c50c9512b1460b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/bfa61d07a0f841d9a77a04e039b8ab5b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/8ec13f8a84334944a5eae2990e3e8bb7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=48.7 K 2024-12-09T06:17:47,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742331_1507 (size=12301) 2024-12-09T06:17:47,360 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e85e61964a44baf88db4675f7a9ca5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733725063213 2024-12-09T06:17:47,360 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585a37b9f61d49e192c50c9512b1460b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733725064419 2024-12-09T06:17:47,360 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa61d07a0f841d9a77a04e039b8ab5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733725064567 2024-12-09T06:17:47,360 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ec13f8a84334944a5eae2990e3e8bb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733725066773 2024-12-09T06:17:47,366 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/1ed1d6ab4dd94d1f97bdf058e4fb291e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1ed1d6ab4dd94d1f97bdf058e4fb291e 2024-12-09T06:17:47,367 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:47,368 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/fa774f1887a34ee1a6bedb5a38677f19 is 50, key is test_row_0/C:col10/1733725066784/Put/seqid=0 2024-12-09T06:17:47,369 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into 1ed1d6ab4dd94d1f97bdf058e4fb291e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:47,369 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:47,369 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=12, startTime=1733725066920; duration=0sec 2024-12-09T06:17:47,369 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:47,369 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:47,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742332_1508 (size=13051) 2024-12-09T06:17:47,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725127417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725127417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725127417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725127727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725127727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725127736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:47,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/f73169d6e7684ec1807b09ec0757f31b 2024-12-09T06:17:47,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/17657fe93104455ab34a590cfee80614 is 50, key is test_row_0/C:col10/1733725066922/Put/seqid=0 2024-12-09T06:17:47,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742333_1509 (size=12301) 2024-12-09T06:17:47,775 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/fa774f1887a34ee1a6bedb5a38677f19 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fa774f1887a34ee1a6bedb5a38677f19 2024-12-09T06:17:47,786 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into fa774f1887a34ee1a6bedb5a38677f19(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:47,786 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:47,786 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=12, startTime=1733725066920; duration=0sec 2024-12-09T06:17:47,787 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:47,787 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T06:17:47,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:47,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:47,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:47,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/17657fe93104455ab34a590cfee80614 2024-12-09T06:17:48,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/abd61548f46441859e19eaa8881b06d9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/abd61548f46441859e19eaa8881b06d9 2024-12-09T06:17:48,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/abd61548f46441859e19eaa8881b06d9, entries=200, sequenceid=354, filesize=14.4 K 2024-12-09T06:17:48,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/f73169d6e7684ec1807b09ec0757f31b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f73169d6e7684ec1807b09ec0757f31b 2024-12-09T06:17:48,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f73169d6e7684ec1807b09ec0757f31b, entries=150, sequenceid=354, filesize=12.0 K 2024-12-09T06:17:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/17657fe93104455ab34a590cfee80614 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17657fe93104455ab34a590cfee80614 2024-12-09T06:17:48,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17657fe93104455ab34a590cfee80614, entries=150, sequenceid=354, filesize=12.0 K 2024-12-09T06:17:48,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~161.11 KB/164976, currentSize=129.71 KB/132820 for c69d9e9023ef26767a3f09a43f2c05f1 in 1268ms, sequenceid=354, compaction requested=false 2024-12-09T06:17:48,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:48,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-12-09T06:17:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:48,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/54e6682627f745b1be09c59769ea55ca is 50, key is test_row_0/A:col10/1733725067078/Put/seqid=0 2024-12-09T06:17:48,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742334_1510 (size=14741) 2024-12-09T06:17:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725128271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725128271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725128272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725128376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725128376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725128386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:48,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725128586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725128586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725128596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:48,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:48,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:48,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:48,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/54e6682627f745b1be09c59769ea55ca 2024-12-09T06:17:48,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/074d2a2d1bba4ad2acc583b8cb0e9516 is 50, key is test_row_0/B:col10/1733725067078/Put/seqid=0 2024-12-09T06:17:48,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742335_1511 (size=12301) 2024-12-09T06:17:48,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/074d2a2d1bba4ad2acc583b8cb0e9516 2024-12-09T06:17:48,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f is 50, key is test_row_0/C:col10/1733725067078/Put/seqid=0 2024-12-09T06:17:48,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742336_1512 (size=12301) 2024-12-09T06:17:48,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f 2024-12-09T06:17:48,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/54e6682627f745b1be09c59769ea55ca as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/54e6682627f745b1be09c59769ea55ca 2024-12-09T06:17:48,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/54e6682627f745b1be09c59769ea55ca, entries=200, sequenceid=393, filesize=14.4 K 2024-12-09T06:17:48,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/074d2a2d1bba4ad2acc583b8cb0e9516 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/074d2a2d1bba4ad2acc583b8cb0e9516 2024-12-09T06:17:48,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/074d2a2d1bba4ad2acc583b8cb0e9516, entries=150, sequenceid=393, filesize=12.0 K 2024-12-09T06:17:48,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f 2024-12-09T06:17:48,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f, entries=150, sequenceid=393, filesize=12.0 K 2024-12-09T06:17:48,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for c69d9e9023ef26767a3f09a43f2c05f1 in 513ms, sequenceid=393, compaction requested=true 2024-12-09T06:17:48,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:48,765 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:48,766 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:48,766 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:48,766 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,766 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fe5623d8681d46fca2f9b47fc4a1338b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/abd61548f46441859e19eaa8881b06d9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/54e6682627f745b1be09c59769ea55ca] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=41.5 K 2024-12-09T06:17:48,766 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe5623d8681d46fca2f9b47fc4a1338b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733725066773 2024-12-09T06:17:48,766 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting abd61548f46441859e19eaa8881b06d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733725066845 2024-12-09T06:17:48,767 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54e6682627f745b1be09c59769ea55ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733725067078 2024-12-09T06:17:48,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:48,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:48,767 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:48,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:48,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:48,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:48,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:48,768 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:48,769 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:48,769 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,769 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1ed1d6ab4dd94d1f97bdf058e4fb291e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f73169d6e7684ec1807b09ec0757f31b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/074d2a2d1bba4ad2acc583b8cb0e9516] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=36.8 K 2024-12-09T06:17:48,769 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ed1d6ab4dd94d1f97bdf058e4fb291e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733725066773 2024-12-09T06:17:48,769 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting f73169d6e7684ec1807b09ec0757f31b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733725066916 2024-12-09T06:17:48,770 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 074d2a2d1bba4ad2acc583b8cb0e9516, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733725067078 2024-12-09T06:17:48,778 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:48,778 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0655f37d9b13491299557e43862bff42 is 50, key is test_row_0/A:col10/1733725067078/Put/seqid=0 2024-12-09T06:17:48,780 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:48,780 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/5e15458959b549da953b994a206e50cd is 50, key is test_row_0/B:col10/1733725067078/Put/seqid=0 2024-12-09T06:17:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T06:17:48,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T06:17:48,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,797 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:17:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:48,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0b594fca14884928b0a6647513fbaa8c is 50, key is test_row_0/A:col10/1733725068271/Put/seqid=0 2024-12-09T06:17:48,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742338_1514 (size=13153) 2024-12-09T06:17:48,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742337_1513 (size=13153) 2024-12-09T06:17:48,830 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0655f37d9b13491299557e43862bff42 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0655f37d9b13491299557e43862bff42 2024-12-09T06:17:48,835 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into 0655f37d9b13491299557e43862bff42(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:48,835 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:48,836 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725068765; duration=0sec 2024-12-09T06:17:48,836 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:48,836 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:48,836 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:48,837 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:48,837 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:48,837 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:48,837 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fa774f1887a34ee1a6bedb5a38677f19, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17657fe93104455ab34a590cfee80614, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=36.8 K 2024-12-09T06:17:48,837 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa774f1887a34ee1a6bedb5a38677f19, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733725066773 2024-12-09T06:17:48,837 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17657fe93104455ab34a590cfee80614, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733725066916 2024-12-09T06:17:48,838 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0522e7a7b8cb4a759ec2e8bd6d9ad54f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733725067078 2024-12-09T06:17:48,848 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:48,848 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/680af3122d6c4981b2d0f9cb56f0464c is 50, key is test_row_0/C:col10/1733725067078/Put/seqid=0 2024-12-09T06:17:48,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742339_1515 (size=9857) 2024-12-09T06:17:48,850 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0b594fca14884928b0a6647513fbaa8c 2024-12-09T06:17:48,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9c397535004842d7bfcfecfb403ae5b5 is 50, key is test_row_0/B:col10/1733725068271/Put/seqid=0 2024-12-09T06:17:48,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742341_1517 (size=9857) 2024-12-09T06:17:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:48,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:48,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742340_1516 (size=13153) 2024-12-09T06:17:48,914 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/680af3122d6c4981b2d0f9cb56f0464c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/680af3122d6c4981b2d0f9cb56f0464c 2024-12-09T06:17:48,919 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into 680af3122d6c4981b2d0f9cb56f0464c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:48,919 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:48,919 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725068767; duration=0sec 2024-12-09T06:17:48,919 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:48,919 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:48,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725068941, max timestamp=1733725068953Num uniques 859; 2024-12-09T06:17:48,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:48,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725068941, max timestamp=1733725068953Num uniques 859; 2024-12-09T06:17:48,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:17:48,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725068941, max timestamp=1733725068953Num uniques 859; 2024-12-09T06:17:48,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:17:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725129014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725069010, max timestamp=1733725069026Num uniques 1018; 2024-12-09T06:17:49,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:17:49,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725069010, max timestamp=1733725069026Num uniques 1018; 2024-12-09T06:17:49,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:17:49,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725069010, max timestamp=1733725069026Num uniques 1009; 2024-12-09T06:17:49,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-12-09T06:17:49,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725129097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725129103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725129137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725129217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,226 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/5e15458959b549da953b994a206e50cd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e15458959b549da953b994a206e50cd 2024-12-09T06:17:49,233 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into 5e15458959b549da953b994a206e50cd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:49,233 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:49,233 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725068767; duration=0sec 2024-12-09T06:17:49,234 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:49,234 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:49,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725129226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,293 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9c397535004842d7bfcfecfb403ae5b5 2024-12-09T06:17:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/fccbe4b8e97b4478865fc076c699d9d5 is 50, key is test_row_0/C:col10/1733725068271/Put/seqid=0 2024-12-09T06:17:49,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742342_1518 (size=9857) 2024-12-09T06:17:49,322 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/fccbe4b8e97b4478865fc076c699d9d5 2024-12-09T06:17:49,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/0b594fca14884928b0a6647513fbaa8c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b594fca14884928b0a6647513fbaa8c 2024-12-09T06:17:49,331 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b594fca14884928b0a6647513fbaa8c, entries=100, sequenceid=403, filesize=9.6 K 2024-12-09T06:17:49,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/9c397535004842d7bfcfecfb403ae5b5 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9c397535004842d7bfcfecfb403ae5b5 2024-12-09T06:17:49,335 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9c397535004842d7bfcfecfb403ae5b5, entries=100, sequenceid=403, filesize=9.6 K 2024-12-09T06:17:49,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/fccbe4b8e97b4478865fc076c699d9d5 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fccbe4b8e97b4478865fc076c699d9d5 2024-12-09T06:17:49,342 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fccbe4b8e97b4478865fc076c699d9d5, entries=100, sequenceid=403, filesize=9.6 K 2024-12-09T06:17:49,343 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c69d9e9023ef26767a3f09a43f2c05f1 in 545ms, sequenceid=403, compaction requested=false 2024-12-09T06:17:49,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:49,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:49,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-09T06:17:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-09T06:17:49,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-09T06:17:49,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6940 sec 2024-12-09T06:17:49,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.7020 sec 2024-12-09T06:17:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:49,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-12-09T06:17:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:49,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/b4312234ce12413081a49875fa8c3ccc is 50, key is test_row_0/A:col10/1733725069095/Put/seqid=0 2024-12-09T06:17:49,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742343_1519 (size=14741) 2024-12-09T06:17:49,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/b4312234ce12413081a49875fa8c3ccc 2024-12-09T06:17:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725129426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/183a153f1c5949fbbddd20318206b646 is 50, key is test_row_0/B:col10/1733725069095/Put/seqid=0 2024-12-09T06:17:49,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725129440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725129447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742344_1520 (size=12301) 2024-12-09T06:17:49,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/183a153f1c5949fbbddd20318206b646 2024-12-09T06:17:49,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/7e302ab4c8b04b2c9fa9e590691684f9 is 50, key is test_row_0/C:col10/1733725069095/Put/seqid=0 2024-12-09T06:17:49,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742345_1521 (size=12301) 2024-12-09T06:17:49,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725129556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725129746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725129767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725129767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:49,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/7e302ab4c8b04b2c9fa9e590691684f9 2024-12-09T06:17:49,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/b4312234ce12413081a49875fa8c3ccc as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/b4312234ce12413081a49875fa8c3ccc 2024-12-09T06:17:49,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/b4312234ce12413081a49875fa8c3ccc, entries=200, sequenceid=460, filesize=14.4 K 2024-12-09T06:17:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/183a153f1c5949fbbddd20318206b646 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/183a153f1c5949fbbddd20318206b646 2024-12-09T06:17:49,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/183a153f1c5949fbbddd20318206b646, entries=150, sequenceid=460, filesize=12.0 K 2024-12-09T06:17:49,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/7e302ab4c8b04b2c9fa9e590691684f9 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/7e302ab4c8b04b2c9fa9e590691684f9 2024-12-09T06:17:49,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/7e302ab4c8b04b2c9fa9e590691684f9, entries=150, sequenceid=460, filesize=12.0 K 2024-12-09T06:17:49,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for c69d9e9023ef26767a3f09a43f2c05f1 in 587ms, sequenceid=460, compaction requested=true 2024-12-09T06:17:49,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:49,954 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:49,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:49,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:49,954 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:49,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:49,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:49,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:49,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:49,956 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:49,956 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:49,956 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:49,956 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e15458959b549da953b994a206e50cd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9c397535004842d7bfcfecfb403ae5b5, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/183a153f1c5949fbbddd20318206b646] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=34.5 K 2024-12-09T06:17:49,956 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e15458959b549da953b994a206e50cd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733725067078 2024-12-09T06:17:49,957 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c397535004842d7bfcfecfb403ae5b5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733725068270 2024-12-09T06:17:49,957 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 183a153f1c5949fbbddd20318206b646, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733725069095 2024-12-09T06:17:49,957 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:49,957 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:49,957 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:49,957 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0655f37d9b13491299557e43862bff42, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b594fca14884928b0a6647513fbaa8c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/b4312234ce12413081a49875fa8c3ccc] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=36.9 K 2024-12-09T06:17:49,958 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0655f37d9b13491299557e43862bff42, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733725067078 2024-12-09T06:17:49,958 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b594fca14884928b0a6647513fbaa8c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733725068270 2024-12-09T06:17:49,958 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4312234ce12413081a49875fa8c3ccc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733725069094 2024-12-09T06:17:49,967 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:49,967 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/ac007115991d4e588f9fcc10196bcb17 is 50, key is test_row_0/B:col10/1733725069095/Put/seqid=0 2024-12-09T06:17:49,974 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:49,975 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/ec3c15e4ea50498a9246b7f14eceac52 is 50, key is test_row_0/A:col10/1733725069095/Put/seqid=0 2024-12-09T06:17:49,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742346_1522 (size=13255) 2024-12-09T06:17:49,991 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/ac007115991d4e588f9fcc10196bcb17 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ac007115991d4e588f9fcc10196bcb17 2024-12-09T06:17:49,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742347_1523 (size=13255) 2024-12-09T06:17:49,996 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into ac007115991d4e588f9fcc10196bcb17(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:49,996 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:49,996 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725069954; duration=0sec 2024-12-09T06:17:49,997 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:49,997 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:49,997 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:49,999 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/ec3c15e4ea50498a9246b7f14eceac52 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ec3c15e4ea50498a9246b7f14eceac52 2024-12-09T06:17:50,000 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:50,000 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:50,000 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:50,000 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/680af3122d6c4981b2d0f9cb56f0464c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fccbe4b8e97b4478865fc076c699d9d5, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/7e302ab4c8b04b2c9fa9e590691684f9] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=34.5 K 2024-12-09T06:17:50,000 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 680af3122d6c4981b2d0f9cb56f0464c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733725067078 2024-12-09T06:17:50,000 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting fccbe4b8e97b4478865fc076c699d9d5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733725068270 2024-12-09T06:17:50,001 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e302ab4c8b04b2c9fa9e590691684f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733725069095 2024-12-09T06:17:50,004 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into ec3c15e4ea50498a9246b7f14eceac52(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:50,004 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:50,004 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725069953; duration=0sec 2024-12-09T06:17:50,004 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:50,004 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:50,008 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:50,009 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/b0ec7725f71a4ce0b6dcbc94a3d27e5b is 50, key is test_row_0/C:col10/1733725069095/Put/seqid=0 2024-12-09T06:17:50,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742348_1524 (size=13255) 2024-12-09T06:17:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:50,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:17:50,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:50,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:50,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:50,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:50,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:50,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:50,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/e00018e334fb4d70919821bbfea9553b is 50, key is test_row_0/A:col10/1733725069419/Put/seqid=0 2024-12-09T06:17:50,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742349_1525 (size=14741) 2024-12-09T06:17:50,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/e00018e334fb4d70919821bbfea9553b 2024-12-09T06:17:50,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/664169047d40408b940c1926d75324e3 is 50, key is test_row_0/B:col10/1733725069419/Put/seqid=0 2024-12-09T06:17:50,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742350_1526 (size=12301) 2024-12-09T06:17:50,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/664169047d40408b940c1926d75324e3 2024-12-09T06:17:50,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/18c3e2a4aea44dcaad13bebedf98dbea is 50, key is test_row_0/C:col10/1733725069419/Put/seqid=0 2024-12-09T06:17:50,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742351_1527 (size=12301) 2024-12-09T06:17:50,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/18c3e2a4aea44dcaad13bebedf98dbea 2024-12-09T06:17:50,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/e00018e334fb4d70919821bbfea9553b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e00018e334fb4d70919821bbfea9553b 2024-12-09T06:17:50,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e00018e334fb4d70919821bbfea9553b, entries=200, sequenceid=472, filesize=14.4 K 2024-12-09T06:17:50,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/664169047d40408b940c1926d75324e3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/664169047d40408b940c1926d75324e3 2024-12-09T06:17:50,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/664169047d40408b940c1926d75324e3, entries=150, sequenceid=472, filesize=12.0 K 2024-12-09T06:17:50,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/18c3e2a4aea44dcaad13bebedf98dbea as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/18c3e2a4aea44dcaad13bebedf98dbea 2024-12-09T06:17:50,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/18c3e2a4aea44dcaad13bebedf98dbea, entries=150, sequenceid=472, filesize=12.0 K 2024-12-09T06:17:50,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=60.38 KB/61830 for c69d9e9023ef26767a3f09a43f2c05f1 in 64ms, sequenceid=472, compaction requested=false 2024-12-09T06:17:50,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:50,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T06:17:50,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:50,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:50,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:50,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:50,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:50,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:50,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/fbfac3b4667040a3be52ea19694175bd is 50, key is test_row_0/A:col10/1733725070146/Put/seqid=0 2024-12-09T06:17:50,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742352_1528 (size=14741) 2024-12-09T06:17:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725130259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725130263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725130287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725130376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725130376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,426 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/b0ec7725f71a4ce0b6dcbc94a3d27e5b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b0ec7725f71a4ce0b6dcbc94a3d27e5b 2024-12-09T06:17:50,429 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into b0ec7725f71a4ce0b6dcbc94a3d27e5b(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:50,429 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:50,429 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725069954; duration=0sec 2024-12-09T06:17:50,429 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:50,429 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:50,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/fbfac3b4667040a3be52ea19694175bd 2024-12-09T06:17:50,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/d137899443234be7908a08daa22412bd is 50, key is test_row_0/B:col10/1733725070146/Put/seqid=0 2024-12-09T06:17:50,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742353_1529 (size=12301) 2024-12-09T06:17:50,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725130586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725130586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T06:17:50,817 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:50,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-09T06:17:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T06:17:50,819 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:50,819 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:50,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:50,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59504 deadline: 1733725130897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59486 deadline: 1733725130897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T06:17:50,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T06:17:50,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:50,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:50,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:50,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:50,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:50,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/d137899443234be7908a08daa22412bd 2024-12-09T06:17:50,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/b6de676eb7424fc38104b9a0046b0b5c is 50, key is test_row_0/C:col10/1733725070146/Put/seqid=0 2024-12-09T06:17:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742354_1530 (size=12301) 2024-12-09T06:17:51,095 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-12-09T06:17:51,095 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:51,095 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,095 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:51,096 INFO [Thread-1937 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1937. 2024-12-09T06:17:51,096 DEBUG [Thread-1937 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:51,096 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,096 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,096 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:51,099 INFO [Thread-1939 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1939. 2024-12-09T06:17:51,099 DEBUG [Thread-1939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:51,099 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,099 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:51,102 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-12-09T06:17:51,102 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-12-09T06:17:51,102 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:51,102 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,102 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:51,102 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,102 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,102 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:51,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:51,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T06:17:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T06:17:51,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T06:17:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59452 deadline: 1733725131306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:51,308 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in c69d9e9023ef26767a3f09a43f2c05f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:17:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 , the old value is region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., hostname=82ac79f0323b,42797,1733724930590, seqNum=2 because the exception is null or not the one we care about 2024-12-09T06:17:51,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/b6de676eb7424fc38104b9a0046b0b5c 2024-12-09T06:17:51,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/fbfac3b4667040a3be52ea19694175bd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fbfac3b4667040a3be52ea19694175bd 2024-12-09T06:17:51,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fbfac3b4667040a3be52ea19694175bd, entries=200, sequenceid=485, filesize=14.4 K 2024-12-09T06:17:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/d137899443234be7908a08daa22412bd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/d137899443234be7908a08daa22412bd 2024-12-09T06:17:51,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/d137899443234be7908a08daa22412bd, entries=150, sequenceid=485, filesize=12.0 K 2024-12-09T06:17:51,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/b6de676eb7424fc38104b9a0046b0b5c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b6de676eb7424fc38104b9a0046b0b5c 2024-12-09T06:17:51,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b6de676eb7424fc38104b9a0046b0b5c, entries=150, sequenceid=485, filesize=12.0 K 2024-12-09T06:17:51,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c69d9e9023ef26767a3f09a43f2c05f1 in 1253ms, sequenceid=485, compaction requested=true 2024-12-09T06:17:51,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:17:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:51,408 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:17:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c69d9e9023ef26767a3f09a43f2c05f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:17:51,408 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:51,408 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:51,409 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/A is initiating minor compaction (all files) 2024-12-09T06:17:51,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T06:17:51,409 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:51,409 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/A in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,409 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/B is initiating minor compaction (all files) 2024-12-09T06:17:51,409 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ec3c15e4ea50498a9246b7f14eceac52, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e00018e334fb4d70919821bbfea9553b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fbfac3b4667040a3be52ea19694175bd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=41.7 K 2024-12-09T06:17:51,409 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/B in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:51,409 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-12-09T06:17:51,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:51,409 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ac007115991d4e588f9fcc10196bcb17, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/664169047d40408b940c1926d75324e3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/d137899443234be7908a08daa22412bd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=37.0 K 2024-12-09T06:17:51,409 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:51,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:51,409 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:51,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:51,409 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:51,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:51,409 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ac007115991d4e588f9fcc10196bcb17, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733725069095 2024-12-09T06:17:51,410 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec3c15e4ea50498a9246b7f14eceac52, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733725069095 2024-12-09T06:17:51,410 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 664169047d40408b940c1926d75324e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733725069383 2024-12-09T06:17:51,410 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting e00018e334fb4d70919821bbfea9553b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733725069383 2024-12-09T06:17:51,410 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting d137899443234be7908a08daa22412bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733725070114 2024-12-09T06:17:51,410 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbfac3b4667040a3be52ea19694175bd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733725070114 2024-12-09T06:17:51,411 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-12-09T06:17:51,411 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:51,411 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,411 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:51,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:51,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/5f5cc440a7054723b491bea9c444fd32 is 50, key is test_row_0/A:col10/1733725070241/Put/seqid=0 2024-12-09T06:17:51,416 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:51,416 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/c2567b18aa6e4c1bb938cd88604c4ba3 is 50, key is test_row_0/B:col10/1733725070146/Put/seqid=0 2024-12-09T06:17:51,416 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#A#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:51,417 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/d6345854ffab47098f996c4ec789c17a is 50, key is test_row_0/A:col10/1733725070146/Put/seqid=0 2024-12-09T06:17:51,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742355_1531 (size=12301) 2024-12-09T06:17:51,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742356_1532 (size=13357) 2024-12-09T06:17:51,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742357_1533 (size=13357) 2024-12-09T06:17:51,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T06:17:51,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:51,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T06:17:51,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T06:17:51,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:51,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T06:17:51,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. as already flushing 2024-12-09T06:17:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:51,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/5f5cc440a7054723b491bea9c444fd32 2024-12-09T06:17:51,828 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/c2567b18aa6e4c1bb938cd88604c4ba3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/c2567b18aa6e4c1bb938cd88604c4ba3 2024-12-09T06:17:51,828 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/d6345854ffab47098f996c4ec789c17a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d6345854ffab47098f996c4ec789c17a 2024-12-09T06:17:51,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/78ccb5480e7847b8bb8c8083d5f46fde is 50, key is test_row_0/B:col10/1733725070241/Put/seqid=0 2024-12-09T06:17:51,832 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/B of c69d9e9023ef26767a3f09a43f2c05f1 into c2567b18aa6e4c1bb938cd88604c4ba3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:51,832 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/A of c69d9e9023ef26767a3f09a43f2c05f1 into d6345854ffab47098f996c4ec789c17a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:51,832 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:51,832 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:51,832 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/B, priority=13, startTime=1733725071408; duration=0sec 2024-12-09T06:17:51,832 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/A, priority=13, startTime=1733725071407; duration=0sec 2024-12-09T06:17:51,833 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:17:51,833 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:B 2024-12-09T06:17:51,833 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:51,833 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:A 2024-12-09T06:17:51,833 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:17:51,833 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:17:51,833 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): c69d9e9023ef26767a3f09a43f2c05f1/C is initiating minor compaction (all files) 2024-12-09T06:17:51,834 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c69d9e9023ef26767a3f09a43f2c05f1/C in TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,834 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b0ec7725f71a4ce0b6dcbc94a3d27e5b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/18c3e2a4aea44dcaad13bebedf98dbea, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b6de676eb7424fc38104b9a0046b0b5c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp, totalSize=37.0 K 2024-12-09T06:17:51,834 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ec7725f71a4ce0b6dcbc94a3d27e5b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733725069095 2024-12-09T06:17:51,834 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c3e2a4aea44dcaad13bebedf98dbea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1733725069383 2024-12-09T06:17:51,834 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b6de676eb7424fc38104b9a0046b0b5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733725070114 2024-12-09T06:17:51,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742358_1534 (size=12301) 2024-12-09T06:17:51,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/78ccb5480e7847b8bb8c8083d5f46fde 2024-12-09T06:17:51,840 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c69d9e9023ef26767a3f09a43f2c05f1#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:17:51,840 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/5a39daa25eaa4cbca76c184bdb0d6987 is 50, key is test_row_0/C:col10/1733725070146/Put/seqid=0 2024-12-09T06:17:51,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/5f9ed6ceac3244ea8c636866e498ed22 is 50, key is test_row_0/C:col10/1733725070241/Put/seqid=0 2024-12-09T06:17:51,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742359_1535 (size=13357) 2024-12-09T06:17:51,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742360_1536 (size=12301) 2024-12-09T06:17:51,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/5f9ed6ceac3244ea8c636866e498ed22 2024-12-09T06:17:51,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/5f5cc440a7054723b491bea9c444fd32 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5f5cc440a7054723b491bea9c444fd32 2024-12-09T06:17:51,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5f5cc440a7054723b491bea9c444fd32, entries=150, sequenceid=507, filesize=12.0 K 2024-12-09T06:17:51,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/78ccb5480e7847b8bb8c8083d5f46fde as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/78ccb5480e7847b8bb8c8083d5f46fde 2024-12-09T06:17:51,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/78ccb5480e7847b8bb8c8083d5f46fde, entries=150, sequenceid=507, filesize=12.0 K 2024-12-09T06:17:51,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/5f9ed6ceac3244ea8c636866e498ed22 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5f9ed6ceac3244ea8c636866e498ed22 2024-12-09T06:17:51,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5f9ed6ceac3244ea8c636866e498ed22, entries=150, sequenceid=507, filesize=12.0 K 2024-12-09T06:17:51,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for c69d9e9023ef26767a3f09a43f2c05f1 in 445ms, sequenceid=507, compaction requested=false 2024-12-09T06:17:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:51,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T06:17:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-09T06:17:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-09T06:17:51,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-09T06:17:51,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0830 sec 2024-12-09T06:17:51,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0890 sec 2024-12-09T06:17:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T06:17:51,957 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:52,254 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/5a39daa25eaa4cbca76c184bdb0d6987 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5a39daa25eaa4cbca76c184bdb0d6987 2024-12-09T06:17:52,258 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c69d9e9023ef26767a3f09a43f2c05f1/C of c69d9e9023ef26767a3f09a43f2c05f1 into 5a39daa25eaa4cbca76c184bdb0d6987(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:17:52,258 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c69d9e9023ef26767a3f09a43f2c05f1: 2024-12-09T06:17:52,259 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., storeName=c69d9e9023ef26767a3f09a43f2c05f1/C, priority=13, startTime=1733725071408; duration=0sec 2024-12-09T06:17:52,259 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:17:52,259 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c69d9e9023ef26767a3f09a43f2c05f1:C 2024-12-09T06:17:53,332 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-12-09T06:17:53,332 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:53,332 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:53,332 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:53,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:53,590 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-12-09T06:17:53,591 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:53,591 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:53,591 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:53,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:53,661 INFO [Thread-1926 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1926. 2024-12-09T06:17:53,661 DEBUG [Thread-1926 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:17:53,661 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:53,661 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:53,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1432 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4292 rows 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1411 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4233 rows 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1432 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4292 rows 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1424 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4271 rows 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1422 2024-12-09T06:17:53,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4264 rows 2024-12-09T06:17:53,663 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T06:17:53,663 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:17:53,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:53,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:53,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:17:53,665 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T06:17:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T06:17:53,669 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725073669"}]},"ts":"1733725073669"} 2024-12-09T06:17:53,670 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T06:17:53,670 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T06:17:53,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:17:53,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, UNASSIGN}] 2024-12-09T06:17:53,673 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, UNASSIGN 2024-12-09T06:17:53,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=c69d9e9023ef26767a3f09a43f2c05f1, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:53,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, UNASSIGN because future has completed 2024-12-09T06:17:53,675 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:17:53,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T06:17:53,827 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:53,827 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:17:53,828 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing c69d9e9023ef26767a3f09a43f2c05f1, disabling compactions & flushes 2024-12-09T06:17:53,828 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:53,828 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:53,828 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. after waiting 0 ms 2024-12-09T06:17:53,828 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:53,828 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing c69d9e9023ef26767a3f09a43f2c05f1 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-09T06:17:53,829 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=A 2024-12-09T06:17:53,829 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:53,829 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=B 2024-12-09T06:17:53,829 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:53,829 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c69d9e9023ef26767a3f09a43f2c05f1, store=C 2024-12-09T06:17:53,829 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:53,835 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/e5af97e047434b9e9cf4eb107d74b963 is 50, key is test_row_0/A:col10/1733725073658/Put/seqid=0 2024-12-09T06:17:53,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742361_1537 (size=9857) 2024-12-09T06:17:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T06:17:54,241 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/e5af97e047434b9e9cf4eb107d74b963 2024-12-09T06:17:54,254 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/b8a9d18b24374a18ad0d70d0f166a975 is 50, key is test_row_0/B:col10/1733725073658/Put/seqid=0 2024-12-09T06:17:54,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742362_1538 (size=9857) 2024-12-09T06:17:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T06:17:54,664 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/b8a9d18b24374a18ad0d70d0f166a975 2024-12-09T06:17:54,673 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/f452a057b8b04de4a95bb198b477cffe is 50, key is test_row_0/C:col10/1733725073658/Put/seqid=0 2024-12-09T06:17:54,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742363_1539 (size=9857) 2024-12-09T06:17:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T06:17:55,076 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/f452a057b8b04de4a95bb198b477cffe 2024-12-09T06:17:55,080 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/A/e5af97e047434b9e9cf4eb107d74b963 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e5af97e047434b9e9cf4eb107d74b963 2024-12-09T06:17:55,083 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e5af97e047434b9e9cf4eb107d74b963, entries=100, sequenceid=517, filesize=9.6 K 2024-12-09T06:17:55,084 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/B/b8a9d18b24374a18ad0d70d0f166a975 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/b8a9d18b24374a18ad0d70d0f166a975 2024-12-09T06:17:55,087 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/b8a9d18b24374a18ad0d70d0f166a975, entries=100, sequenceid=517, filesize=9.6 K 2024-12-09T06:17:55,088 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/.tmp/C/f452a057b8b04de4a95bb198b477cffe as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f452a057b8b04de4a95bb198b477cffe 2024-12-09T06:17:55,091 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f452a057b8b04de4a95bb198b477cffe, entries=100, sequenceid=517, filesize=9.6 K 2024-12-09T06:17:55,092 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c69d9e9023ef26767a3f09a43f2c05f1 in 1264ms, sequenceid=517, compaction requested=true 2024-12-09T06:17:55,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/62dec0fc70ad4ff3a3361b3bd6dcba07, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b5ef684380947d29a0435dbb32eba8f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d912f878469f40ffb1ffc9d600398357, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ff63515bb8d44afb929f5e7479be0d31, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0f0fb84c8ea24a91ae501d01497e30ad, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/29d4c716e5564033b36b4372ba616b27, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/2ccf058c238c47239e6d3574b003c83d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9eedbf9df5c5498085b790c8d36941e8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/3987e2c645644937b47b631b75db3ed4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9599a0a55b88420b9f57a0cdfd328e18, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/702bc97c62fe46b983fe0a132fceae2f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/f0f781b869ce48ab895961676be2ccda, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/38995b8927184bdab370ae63cde7a8dc, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/1d129a5ea7074c4cbc7e11ae9aedb01f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/90abd74af2774f349910c851ade24a85, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9316d49799aa490488994f06fa4a54a0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/49aa615c8b0d4173aca5a48cf215465b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5550c140033a46798d0f306c7f536e87, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8ab0ffac6407461a86caa969af4473d0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/913038cd1e624082b3611e11865e7c37, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/50eb91d0f39b4b5da615c7480e52036f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9953d389468b40a8bab0fc44ebe225d8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fe5623d8681d46fca2f9b47fc4a1338b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/abd61548f46441859e19eaa8881b06d9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/54e6682627f745b1be09c59769ea55ca, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0655f37d9b13491299557e43862bff42, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b594fca14884928b0a6647513fbaa8c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/b4312234ce12413081a49875fa8c3ccc, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ec3c15e4ea50498a9246b7f14eceac52, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e00018e334fb4d70919821bbfea9553b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fbfac3b4667040a3be52ea19694175bd] to archive 2024-12-09T06:17:55,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:55,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/62dec0fc70ad4ff3a3361b3bd6dcba07 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/62dec0fc70ad4ff3a3361b3bd6dcba07 2024-12-09T06:17:55,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b5ef684380947d29a0435dbb32eba8f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b5ef684380947d29a0435dbb32eba8f 2024-12-09T06:17:55,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d912f878469f40ffb1ffc9d600398357 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d912f878469f40ffb1ffc9d600398357 2024-12-09T06:17:55,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ff63515bb8d44afb929f5e7479be0d31 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ff63515bb8d44afb929f5e7479be0d31 2024-12-09T06:17:55,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0f0fb84c8ea24a91ae501d01497e30ad to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0f0fb84c8ea24a91ae501d01497e30ad 2024-12-09T06:17:55,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/29d4c716e5564033b36b4372ba616b27 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/29d4c716e5564033b36b4372ba616b27 2024-12-09T06:17:55,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/2ccf058c238c47239e6d3574b003c83d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/2ccf058c238c47239e6d3574b003c83d 2024-12-09T06:17:55,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9eedbf9df5c5498085b790c8d36941e8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9eedbf9df5c5498085b790c8d36941e8 2024-12-09T06:17:55,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/3987e2c645644937b47b631b75db3ed4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/3987e2c645644937b47b631b75db3ed4 2024-12-09T06:17:55,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9599a0a55b88420b9f57a0cdfd328e18 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9599a0a55b88420b9f57a0cdfd328e18 2024-12-09T06:17:55,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/702bc97c62fe46b983fe0a132fceae2f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/702bc97c62fe46b983fe0a132fceae2f 2024-12-09T06:17:55,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/f0f781b869ce48ab895961676be2ccda to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/f0f781b869ce48ab895961676be2ccda 2024-12-09T06:17:55,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/38995b8927184bdab370ae63cde7a8dc to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/38995b8927184bdab370ae63cde7a8dc 2024-12-09T06:17:55,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/1d129a5ea7074c4cbc7e11ae9aedb01f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/1d129a5ea7074c4cbc7e11ae9aedb01f 2024-12-09T06:17:55,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/90abd74af2774f349910c851ade24a85 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/90abd74af2774f349910c851ade24a85 2024-12-09T06:17:55,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9316d49799aa490488994f06fa4a54a0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9316d49799aa490488994f06fa4a54a0 2024-12-09T06:17:55,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/49aa615c8b0d4173aca5a48cf215465b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/49aa615c8b0d4173aca5a48cf215465b 2024-12-09T06:17:55,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5550c140033a46798d0f306c7f536e87 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5550c140033a46798d0f306c7f536e87 2024-12-09T06:17:55,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8ab0ffac6407461a86caa969af4473d0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/8ab0ffac6407461a86caa969af4473d0 2024-12-09T06:17:55,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/913038cd1e624082b3611e11865e7c37 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/913038cd1e624082b3611e11865e7c37 2024-12-09T06:17:55,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/50eb91d0f39b4b5da615c7480e52036f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/50eb91d0f39b4b5da615c7480e52036f 2024-12-09T06:17:55,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9953d389468b40a8bab0fc44ebe225d8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/9953d389468b40a8bab0fc44ebe225d8 2024-12-09T06:17:55,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fe5623d8681d46fca2f9b47fc4a1338b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fe5623d8681d46fca2f9b47fc4a1338b 2024-12-09T06:17:55,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/abd61548f46441859e19eaa8881b06d9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/abd61548f46441859e19eaa8881b06d9 2024-12-09T06:17:55,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/54e6682627f745b1be09c59769ea55ca to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/54e6682627f745b1be09c59769ea55ca 2024-12-09T06:17:55,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0655f37d9b13491299557e43862bff42 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0655f37d9b13491299557e43862bff42 2024-12-09T06:17:55,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b594fca14884928b0a6647513fbaa8c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/0b594fca14884928b0a6647513fbaa8c 2024-12-09T06:17:55,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/b4312234ce12413081a49875fa8c3ccc to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/b4312234ce12413081a49875fa8c3ccc 2024-12-09T06:17:55,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ec3c15e4ea50498a9246b7f14eceac52 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/ec3c15e4ea50498a9246b7f14eceac52 2024-12-09T06:17:55,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e00018e334fb4d70919821bbfea9553b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e00018e334fb4d70919821bbfea9553b 2024-12-09T06:17:55,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fbfac3b4667040a3be52ea19694175bd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/fbfac3b4667040a3be52ea19694175bd 2024-12-09T06:17:55,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ce7eb35e68034c839976b9e09fea2dc0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9137af71e4d04593b0bf45db9059194b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f02fec71447b4c58aa8cf392a5af885a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/88f3d1e4f12848d986f484dface19b14, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/64f5470ac28446c889ed1608502d1a7e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6a6c75312a449d296fa35faa765a0cb, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/08618ad5f7e7489bb2e2dcf3f00a1667, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e2c6db50eb04b28b811a66acc5a5e02, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8682d2429d02404db28f12012ec8e919, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1103a4b4e93b408d8ee03ccc46342a19, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5b1de720f601475aae10cc3972caf02f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8eefe9db47604759b8e486a808898124, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fbf5b20fdbc142688c1f015a6f8ef54c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/2b32f2660d684f029948a78d95c7219b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/de0d68c6dcc74729872d1c5823cdeef7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/10eecd2142b54442a3c2ced9c551e53c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/02ab8614e39f48768313e2dd612abc38, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/c27f31ad7af64be394848a17fa5436a1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/0ba9f7217e9a47fd94f2c6f9d0993f26, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6f805ad7d3649a1b87f07b346f08a5c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fe9f7db0cb884f1aa0aa5f0143d958a3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1ed1d6ab4dd94d1f97bdf058e4fb291e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/4e803dbc16094c6093fff921088c0a6a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f73169d6e7684ec1807b09ec0757f31b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e15458959b549da953b994a206e50cd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/074d2a2d1bba4ad2acc583b8cb0e9516, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9c397535004842d7bfcfecfb403ae5b5, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ac007115991d4e588f9fcc10196bcb17, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/183a153f1c5949fbbddd20318206b646, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/664169047d40408b940c1926d75324e3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/d137899443234be7908a08daa22412bd] to archive 2024-12-09T06:17:55,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:55,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ce7eb35e68034c839976b9e09fea2dc0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ce7eb35e68034c839976b9e09fea2dc0 2024-12-09T06:17:55,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9137af71e4d04593b0bf45db9059194b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9137af71e4d04593b0bf45db9059194b 2024-12-09T06:17:55,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f02fec71447b4c58aa8cf392a5af885a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f02fec71447b4c58aa8cf392a5af885a 2024-12-09T06:17:55,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/88f3d1e4f12848d986f484dface19b14 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/88f3d1e4f12848d986f484dface19b14 2024-12-09T06:17:55,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/64f5470ac28446c889ed1608502d1a7e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/64f5470ac28446c889ed1608502d1a7e 2024-12-09T06:17:55,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6a6c75312a449d296fa35faa765a0cb to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6a6c75312a449d296fa35faa765a0cb 2024-12-09T06:17:55,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/08618ad5f7e7489bb2e2dcf3f00a1667 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/08618ad5f7e7489bb2e2dcf3f00a1667 2024-12-09T06:17:55,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e2c6db50eb04b28b811a66acc5a5e02 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e2c6db50eb04b28b811a66acc5a5e02 2024-12-09T06:17:55,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8682d2429d02404db28f12012ec8e919 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8682d2429d02404db28f12012ec8e919 2024-12-09T06:17:55,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1103a4b4e93b408d8ee03ccc46342a19 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1103a4b4e93b408d8ee03ccc46342a19 2024-12-09T06:17:55,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5b1de720f601475aae10cc3972caf02f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5b1de720f601475aae10cc3972caf02f 2024-12-09T06:17:55,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8eefe9db47604759b8e486a808898124 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/8eefe9db47604759b8e486a808898124 2024-12-09T06:17:55,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fbf5b20fdbc142688c1f015a6f8ef54c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fbf5b20fdbc142688c1f015a6f8ef54c 2024-12-09T06:17:55,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/2b32f2660d684f029948a78d95c7219b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/2b32f2660d684f029948a78d95c7219b 2024-12-09T06:17:55,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/de0d68c6dcc74729872d1c5823cdeef7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/de0d68c6dcc74729872d1c5823cdeef7 2024-12-09T06:17:55,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/10eecd2142b54442a3c2ced9c551e53c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/10eecd2142b54442a3c2ced9c551e53c 2024-12-09T06:17:55,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/02ab8614e39f48768313e2dd612abc38 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/02ab8614e39f48768313e2dd612abc38 2024-12-09T06:17:55,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/c27f31ad7af64be394848a17fa5436a1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/c27f31ad7af64be394848a17fa5436a1 2024-12-09T06:17:55,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/0ba9f7217e9a47fd94f2c6f9d0993f26 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/0ba9f7217e9a47fd94f2c6f9d0993f26 2024-12-09T06:17:55,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6f805ad7d3649a1b87f07b346f08a5c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/e6f805ad7d3649a1b87f07b346f08a5c 2024-12-09T06:17:55,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fe9f7db0cb884f1aa0aa5f0143d958a3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/fe9f7db0cb884f1aa0aa5f0143d958a3 2024-12-09T06:17:55,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1ed1d6ab4dd94d1f97bdf058e4fb291e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/1ed1d6ab4dd94d1f97bdf058e4fb291e 2024-12-09T06:17:55,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/4e803dbc16094c6093fff921088c0a6a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/4e803dbc16094c6093fff921088c0a6a 2024-12-09T06:17:55,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f73169d6e7684ec1807b09ec0757f31b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/f73169d6e7684ec1807b09ec0757f31b 2024-12-09T06:17:55,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e15458959b549da953b994a206e50cd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/5e15458959b549da953b994a206e50cd 2024-12-09T06:17:55,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/074d2a2d1bba4ad2acc583b8cb0e9516 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/074d2a2d1bba4ad2acc583b8cb0e9516 2024-12-09T06:17:55,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9c397535004842d7bfcfecfb403ae5b5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/9c397535004842d7bfcfecfb403ae5b5 2024-12-09T06:17:55,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ac007115991d4e588f9fcc10196bcb17 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/ac007115991d4e588f9fcc10196bcb17 2024-12-09T06:17:55,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/183a153f1c5949fbbddd20318206b646 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/183a153f1c5949fbbddd20318206b646 2024-12-09T06:17:55,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/664169047d40408b940c1926d75324e3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/664169047d40408b940c1926d75324e3 2024-12-09T06:17:55,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/d137899443234be7908a08daa22412bd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/d137899443234be7908a08daa22412bd 2024-12-09T06:17:55,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/4f43a71651434f1c8591d11f987c1d5b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/6df6b19076b44100b211df3d510958aa, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/d687b403bfa54417bcc01d1ccf66d252, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/606ecfa7736c46beafae99e970358c58, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/3ae08c1fe5f74f73b55edbef74b923b6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/21f4589822b344c19d36d674a6d304da, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/04c28fd89760416d878152175408a12d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/533a018bad9e41489a59f31a4ea8fc65, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b431b91b2a5b4b4f95df6f3a84b5463a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/30ef95ab4fec4f9b8d7d056d10073d8d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/03bac667e12d4c429b4f83a179cc241a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/de57e72c6bb04f22a1565a9082555bd3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/be08b86300024f99b80b502ffb117065, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/848154c843074f2dba8f73215f409f0f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17777a8afbc0461baa7601ee3f1865e1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/c74e9a47dff5427c9761683380053967, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/518d954b359f492783a00015ae1d4943, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/9e85e61964a44baf88db4675f7a9ca5a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5c7aeebfb7644d23a7630d2a879b7d2e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/585a37b9f61d49e192c50c9512b1460b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/bfa61d07a0f841d9a77a04e039b8ab5b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fa774f1887a34ee1a6bedb5a38677f19, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/8ec13f8a84334944a5eae2990e3e8bb7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17657fe93104455ab34a590cfee80614, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/680af3122d6c4981b2d0f9cb56f0464c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fccbe4b8e97b4478865fc076c699d9d5, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b0ec7725f71a4ce0b6dcbc94a3d27e5b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/7e302ab4c8b04b2c9fa9e590691684f9, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/18c3e2a4aea44dcaad13bebedf98dbea, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b6de676eb7424fc38104b9a0046b0b5c] to archive 2024-12-09T06:17:55,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:17:55,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/4f43a71651434f1c8591d11f987c1d5b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/4f43a71651434f1c8591d11f987c1d5b 2024-12-09T06:17:55,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/6df6b19076b44100b211df3d510958aa to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/6df6b19076b44100b211df3d510958aa 2024-12-09T06:17:55,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/d687b403bfa54417bcc01d1ccf66d252 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/d687b403bfa54417bcc01d1ccf66d252 2024-12-09T06:17:55,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/606ecfa7736c46beafae99e970358c58 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/606ecfa7736c46beafae99e970358c58 2024-12-09T06:17:55,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/3ae08c1fe5f74f73b55edbef74b923b6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/3ae08c1fe5f74f73b55edbef74b923b6 2024-12-09T06:17:55,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/21f4589822b344c19d36d674a6d304da to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/21f4589822b344c19d36d674a6d304da 2024-12-09T06:17:55,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/04c28fd89760416d878152175408a12d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/04c28fd89760416d878152175408a12d 2024-12-09T06:17:55,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/533a018bad9e41489a59f31a4ea8fc65 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/533a018bad9e41489a59f31a4ea8fc65 2024-12-09T06:17:55,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b431b91b2a5b4b4f95df6f3a84b5463a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b431b91b2a5b4b4f95df6f3a84b5463a 2024-12-09T06:17:55,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/30ef95ab4fec4f9b8d7d056d10073d8d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/30ef95ab4fec4f9b8d7d056d10073d8d 2024-12-09T06:17:55,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/03bac667e12d4c429b4f83a179cc241a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/03bac667e12d4c429b4f83a179cc241a 2024-12-09T06:17:55,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/de57e72c6bb04f22a1565a9082555bd3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/de57e72c6bb04f22a1565a9082555bd3 2024-12-09T06:17:55,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/be08b86300024f99b80b502ffb117065 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/be08b86300024f99b80b502ffb117065 2024-12-09T06:17:55,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/848154c843074f2dba8f73215f409f0f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/848154c843074f2dba8f73215f409f0f 2024-12-09T06:17:55,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17777a8afbc0461baa7601ee3f1865e1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17777a8afbc0461baa7601ee3f1865e1 2024-12-09T06:17:55,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/c74e9a47dff5427c9761683380053967 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/c74e9a47dff5427c9761683380053967 2024-12-09T06:17:55,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/518d954b359f492783a00015ae1d4943 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/518d954b359f492783a00015ae1d4943 2024-12-09T06:17:55,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/9e85e61964a44baf88db4675f7a9ca5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/9e85e61964a44baf88db4675f7a9ca5a 2024-12-09T06:17:55,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5c7aeebfb7644d23a7630d2a879b7d2e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5c7aeebfb7644d23a7630d2a879b7d2e 2024-12-09T06:17:55,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/585a37b9f61d49e192c50c9512b1460b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/585a37b9f61d49e192c50c9512b1460b 2024-12-09T06:17:55,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/bfa61d07a0f841d9a77a04e039b8ab5b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/bfa61d07a0f841d9a77a04e039b8ab5b 2024-12-09T06:17:55,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fa774f1887a34ee1a6bedb5a38677f19 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fa774f1887a34ee1a6bedb5a38677f19 2024-12-09T06:17:55,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/8ec13f8a84334944a5eae2990e3e8bb7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/8ec13f8a84334944a5eae2990e3e8bb7 2024-12-09T06:17:55,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17657fe93104455ab34a590cfee80614 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/17657fe93104455ab34a590cfee80614 2024-12-09T06:17:55,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/680af3122d6c4981b2d0f9cb56f0464c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/680af3122d6c4981b2d0f9cb56f0464c 2024-12-09T06:17:55,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/0522e7a7b8cb4a759ec2e8bd6d9ad54f 2024-12-09T06:17:55,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fccbe4b8e97b4478865fc076c699d9d5 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/fccbe4b8e97b4478865fc076c699d9d5 2024-12-09T06:17:55,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b0ec7725f71a4ce0b6dcbc94a3d27e5b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b0ec7725f71a4ce0b6dcbc94a3d27e5b 2024-12-09T06:17:55,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/7e302ab4c8b04b2c9fa9e590691684f9 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/7e302ab4c8b04b2c9fa9e590691684f9 2024-12-09T06:17:55,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/18c3e2a4aea44dcaad13bebedf98dbea to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/18c3e2a4aea44dcaad13bebedf98dbea 2024-12-09T06:17:55,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b6de676eb7424fc38104b9a0046b0b5c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/b6de676eb7424fc38104b9a0046b0b5c 2024-12-09T06:17:55,213 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/recovered.edits/520.seqid, newMaxSeqId=520, maxSeqId=1 2024-12-09T06:17:55,215 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. 2024-12-09T06:17:55,215 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for c69d9e9023ef26767a3f09a43f2c05f1: Waiting for close lock at 1733725073828Running coprocessor pre-close hooks at 1733725073828Disabling compacts and flushes for region at 1733725073828Disabling writes for close at 1733725073828Obtaining lock to block concurrent updates at 1733725073828Preparing flush snapshotting stores in c69d9e9023ef26767a3f09a43f2c05f1 at 1733725073829 (+1 ms)Finished memstore snapshotting TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733725073830 (+1 ms)Flushing stores of TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1. at 1733725073831 (+1 ms)Flushing c69d9e9023ef26767a3f09a43f2c05f1/A: creating writer at 1733725073831Flushing c69d9e9023ef26767a3f09a43f2c05f1/A: appending metadata at 1733725073835 (+4 ms)Flushing c69d9e9023ef26767a3f09a43f2c05f1/A: closing flushed file at 1733725073835Flushing c69d9e9023ef26767a3f09a43f2c05f1/B: creating writer at 1733725074250 (+415 ms)Flushing c69d9e9023ef26767a3f09a43f2c05f1/B: appending metadata at 1733725074253 (+3 ms)Flushing c69d9e9023ef26767a3f09a43f2c05f1/B: closing flushed file at 1733725074253Flushing c69d9e9023ef26767a3f09a43f2c05f1/C: creating writer at 1733725074670 (+417 ms)Flushing c69d9e9023ef26767a3f09a43f2c05f1/C: appending metadata at 1733725074672 (+2 ms)Flushing c69d9e9023ef26767a3f09a43f2c05f1/C: closing flushed file at 1733725074672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f53b304: reopening flushed file at 1733725075079 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a2636eb: reopening flushed file at 1733725075083 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17b0411e: reopening flushed file at 1733725075087 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c69d9e9023ef26767a3f09a43f2c05f1 in 1264ms, sequenceid=517, compaction requested=true at 1733725075092 (+5 ms)Writing region close event to WAL at 1733725075207 (+115 ms)Running coprocessor post-close hooks at 1733725075215 (+8 ms)Closed at 1733725075215 2024-12-09T06:17:55,217 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:55,218 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=c69d9e9023ef26767a3f09a43f2c05f1, regionState=CLOSED 2024-12-09T06:17:55,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:55,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T06:17:55,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure c69d9e9023ef26767a3f09a43f2c05f1, server=82ac79f0323b,42797,1733724930590 in 1.5450 sec 2024-12-09T06:17:55,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-09T06:17:55,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c69d9e9023ef26767a3f09a43f2c05f1, UNASSIGN in 1.5490 sec 2024-12-09T06:17:55,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T06:17:55,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5530 sec 2024-12-09T06:17:55,226 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725075226"}]},"ts":"1733725075226"} 2024-12-09T06:17:55,230 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T06:17:55,230 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T06:17:55,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5660 sec 2024-12-09T06:17:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T06:17:55,816 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:55,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T06:17:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:55,819 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T06:17:55,820 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:55,822 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:55,824 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/recovered.edits] 2024-12-09T06:17:55,827 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5f5cc440a7054723b491bea9c444fd32 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/5f5cc440a7054723b491bea9c444fd32 2024-12-09T06:17:55,828 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d6345854ffab47098f996c4ec789c17a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/d6345854ffab47098f996c4ec789c17a 2024-12-09T06:17:55,829 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e5af97e047434b9e9cf4eb107d74b963 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/A/e5af97e047434b9e9cf4eb107d74b963 2024-12-09T06:17:55,831 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/78ccb5480e7847b8bb8c8083d5f46fde to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/78ccb5480e7847b8bb8c8083d5f46fde 2024-12-09T06:17:55,831 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/b8a9d18b24374a18ad0d70d0f166a975 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/b8a9d18b24374a18ad0d70d0f166a975 2024-12-09T06:17:55,832 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/c2567b18aa6e4c1bb938cd88604c4ba3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/B/c2567b18aa6e4c1bb938cd88604c4ba3 2024-12-09T06:17:55,834 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5a39daa25eaa4cbca76c184bdb0d6987 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5a39daa25eaa4cbca76c184bdb0d6987 2024-12-09T06:17:55,835 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5f9ed6ceac3244ea8c636866e498ed22 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/5f9ed6ceac3244ea8c636866e498ed22 2024-12-09T06:17:55,835 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f452a057b8b04de4a95bb198b477cffe to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/C/f452a057b8b04de4a95bb198b477cffe 2024-12-09T06:17:55,837 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/recovered.edits/520.seqid to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1/recovered.edits/520.seqid 2024-12-09T06:17:55,838 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/c69d9e9023ef26767a3f09a43f2c05f1 2024-12-09T06:17:55,838 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T06:17:55,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:55,841 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T06:17:55,844 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T06:17:55,845 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:55,845 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T06:17:55,845 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733725075845"}]},"ts":"9223372036854775807"} 2024-12-09T06:17:55,850 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T06:17:55,850 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c69d9e9023ef26767a3f09a43f2c05f1, NAME => 'TestAcidGuarantees,,1733725048895.c69d9e9023ef26767a3f09a43f2c05f1.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T06:17:55,851 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T06:17:55,851 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733725075851"}]},"ts":"9223372036854775807"} 2024-12-09T06:17:55,853 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T06:17:55,853 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:55,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-12-09T06:17:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T06:17:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T06:17:55,929 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:55,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=455 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=792 (was 919), ProcessCount=11 (was 11), AvailableMemoryMB=5747 (was 5372) - AvailableMemoryMB LEAK? - 2024-12-09T06:17:55,942 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=792, ProcessCount=11, AvailableMemoryMB=5746 2024-12-09T06:17:55,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:17:55,943 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:17:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:55,945 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T06:17:55,945 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:55,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-12-09T06:17:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T06:17:55,946 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T06:17:55,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742364_1540 (size=1061) 2024-12-09T06:17:55,951 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d 2024-12-09T06:17:55,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742365_1541 (size=53) 2024-12-09T06:17:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T06:17:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T06:17:56,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:17:56,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 86ed6927f9c6713f75d4b2e28d186f5a, disabling compactions & flushes 2024-12-09T06:17:56,359 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:56,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:56,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. after waiting 0 ms 2024-12-09T06:17:56,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:56,359 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:56,359 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86ed6927f9c6713f75d4b2e28d186f5a: Waiting for close lock at 1733725076359Disabling compacts and flushes for region at 1733725076359Disabling writes for close at 1733725076359Writing region close event to WAL at 1733725076359Closed at 1733725076359 2024-12-09T06:17:56,360 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T06:17:56,361 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733725076360"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733725076360"}]},"ts":"1733725076360"} 2024-12-09T06:17:56,362 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T06:17:56,363 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T06:17:56,363 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725076363"}]},"ts":"1733725076363"} 2024-12-09T06:17:56,365 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T06:17:56,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, ASSIGN}] 2024-12-09T06:17:56,367 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, ASSIGN 2024-12-09T06:17:56,367 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, ASSIGN; state=OFFLINE, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=false 2024-12-09T06:17:56,518 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=86ed6927f9c6713f75d4b2e28d186f5a, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:56,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, ASSIGN because future has completed 2024-12-09T06:17:56,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T06:17:56,674 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:56,674 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:17:56,675 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,675 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:17:56,675 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,675 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,676 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,677 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:56,677 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86ed6927f9c6713f75d4b2e28d186f5a columnFamilyName A 2024-12-09T06:17:56,677 DEBUG [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:56,677 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(327): Store=86ed6927f9c6713f75d4b2e28d186f5a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:56,678 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,678 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:56,679 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86ed6927f9c6713f75d4b2e28d186f5a columnFamilyName B 2024-12-09T06:17:56,679 DEBUG [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:56,679 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(327): Store=86ed6927f9c6713f75d4b2e28d186f5a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:56,679 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,680 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:56,680 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86ed6927f9c6713f75d4b2e28d186f5a columnFamilyName C 2024-12-09T06:17:56,680 DEBUG [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:56,680 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(327): Store=86ed6927f9c6713f75d4b2e28d186f5a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:56,680 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:56,681 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,681 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,681 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,682 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,682 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,682 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:17:56,683 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,685 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T06:17:56,685 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 86ed6927f9c6713f75d4b2e28d186f5a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60566390, jitterRate=-0.09749045968055725}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:17:56,685 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:56,686 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 86ed6927f9c6713f75d4b2e28d186f5a: Running coprocessor pre-open hook at 1733725076675Writing region info on filesystem at 1733725076675Initializing all the Stores at 1733725076675Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725076675Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725076676 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725076676Cleaning up temporary data from old regions at 1733725076682 (+6 ms)Running coprocessor post-open hooks at 1733725076685 (+3 ms)Region opened successfully at 1733725076686 (+1 ms) 2024-12-09T06:17:56,686 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., pid=140, masterSystemTime=1733725076671 2024-12-09T06:17:56,688 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:56,688 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:56,688 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=86ed6927f9c6713f75d4b2e28d186f5a, regionState=OPEN, openSeqNum=2, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:56,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:56,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-09T06:17:56,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 in 170 msec 2024-12-09T06:17:56,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-09T06:17:56,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, ASSIGN in 327 msec 2024-12-09T06:17:56,695 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T06:17:56,695 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725076695"}]},"ts":"1733725076695"} 2024-12-09T06:17:56,697 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T06:17:56,698 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T06:17:56,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 755 msec 2024-12-09T06:17:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T06:17:57,086 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:57,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:57,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:57,088 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:57,089 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6289fb5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:57,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:57,090 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:57,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355713d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:57,091 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:57,093 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:57,094 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T06:17:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 82ac79f0323b,43241,1733724929416 2024-12-09T06:17:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21aa1aaf 2024-12-09T06:17:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T06:17:57,096 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T06:17:57,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T06:17:57,098 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T06:17:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T06:17:57,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742366_1542 (size=1097) 2024-12-09T06:17:57,519 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T06:17:57,519 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T06:17:57,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:17:57,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, REOPEN/MOVE}] 2024-12-09T06:17:57,525 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, REOPEN/MOVE 2024-12-09T06:17:57,526 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=86ed6927f9c6713f75d4b2e28d186f5a, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:57,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, REOPEN/MOVE because future has completed 2024-12-09T06:17:57,532 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:17:57,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:57,685 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:57,685 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:17:57,685 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 86ed6927f9c6713f75d4b2e28d186f5a, disabling compactions & flushes 2024-12-09T06:17:57,685 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:57,685 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:57,686 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. after waiting 1 ms 2024-12-09T06:17:57,686 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:57,689 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T06:17:57,690 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:57,690 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 86ed6927f9c6713f75d4b2e28d186f5a: Waiting for close lock at 1733725077685Running coprocessor pre-close hooks at 1733725077685Disabling compacts and flushes for region at 1733725077685Disabling writes for close at 1733725077686 (+1 ms)Writing region close event to WAL at 1733725077686Running coprocessor post-close hooks at 1733725077690 (+4 ms)Closed at 1733725077690 2024-12-09T06:17:57,690 WARN [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 86ed6927f9c6713f75d4b2e28d186f5a to self. 2024-12-09T06:17:57,692 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:57,692 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=86ed6927f9c6713f75d4b2e28d186f5a, regionState=CLOSED 2024-12-09T06:17:57,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:57,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-09T06:17:57,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 in 161 msec 2024-12-09T06:17:57,696 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, REOPEN/MOVE; state=CLOSED, location=82ac79f0323b,42797,1733724930590; forceNewPlan=false, retain=true 2024-12-09T06:17:57,846 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=86ed6927f9c6713f75d4b2e28d186f5a, regionState=OPENING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:57,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, REOPEN/MOVE because future has completed 2024-12-09T06:17:57,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:17:58,003 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,003 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T06:17:58,003 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,003 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T06:17:58,004 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,004 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,005 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,005 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:58,005 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86ed6927f9c6713f75d4b2e28d186f5a columnFamilyName A 2024-12-09T06:17:58,006 DEBUG [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:58,007 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(327): Store=86ed6927f9c6713f75d4b2e28d186f5a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:58,007 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,007 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:58,007 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86ed6927f9c6713f75d4b2e28d186f5a columnFamilyName B 2024-12-09T06:17:58,007 DEBUG [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:58,008 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(327): Store=86ed6927f9c6713f75d4b2e28d186f5a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:58,008 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,008 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T06:17:58,008 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86ed6927f9c6713f75d4b2e28d186f5a columnFamilyName C 2024-12-09T06:17:58,008 DEBUG [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:58,009 INFO [StoreOpener-86ed6927f9c6713f75d4b2e28d186f5a-1 {}] regionserver.HStore(327): Store=86ed6927f9c6713f75d4b2e28d186f5a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T06:17:58,009 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,009 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,009 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,010 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,010 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,010 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,011 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T06:17:58,012 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,012 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 86ed6927f9c6713f75d4b2e28d186f5a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70405016, jitterRate=0.04911649227142334}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T06:17:58,012 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,013 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 86ed6927f9c6713f75d4b2e28d186f5a: Running coprocessor pre-open hook at 1733725078004Writing region info on filesystem at 1733725078004Initializing all the Stores at 1733725078004Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725078004Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725078004Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733725078004Cleaning up temporary data from old regions at 1733725078010 (+6 ms)Running coprocessor post-open hooks at 1733725078012 (+2 ms)Region opened successfully at 1733725078013 (+1 ms) 2024-12-09T06:17:58,014 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., pid=145, masterSystemTime=1733725078001 2024-12-09T06:17:58,015 DEBUG [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,015 INFO [RS_OPEN_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,016 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=86ed6927f9c6713f75d4b2e28d186f5a, regionState=OPEN, openSeqNum=5, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:17:58,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-12-09T06:17:58,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 in 168 msec 2024-12-09T06:17:58,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-09T06:17:58,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, REOPEN/MOVE in 495 msec 2024-12-09T06:17:58,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-09T06:17:58,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-12-09T06:17:58,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-12-09T06:17:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-09T06:17:58,024 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:17:58,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c49abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1a04c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,028 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3c7c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,029 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b28fd99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550eb015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@365b17ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b639d86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f76bc4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edd9ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,037 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108360db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,039 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ffc277, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,040 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494d9226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,042 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@98d6bd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,043 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dcfd0b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,045 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4278dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,047 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab7cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,049 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f62158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,051 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@912a340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 82ac79f0323b,43241,-1 for getting cluster id 2024-12-09T06:17:58,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T06:17:58,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '14f39449-667b-4aed-944a-0c1e2e63b424' 2024-12-09T06:17:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T06:17:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "14f39449-667b-4aed-944a-0c1e2e63b424" 2024-12-09T06:17:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d43525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [82ac79f0323b,43241,-1] 2024-12-09T06:17:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T06:17:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:17:58,054 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T06:17:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44700f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T06:17:58,055 DEBUG [Thread-2417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,055 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,055 DEBUG [Thread-2415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:17:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,059 DEBUG [Thread-2428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,060 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-09T06:17:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,061 DEBUG [Thread-2413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,061 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:17:58,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T06:17:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,062 DEBUG [Thread-2424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,063 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,064 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,064 DEBUG [Thread-2426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T06:17:58,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:17:58,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:17:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,065 DEBUG [Thread-2420 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=82ac79f0323b,42797,1733724930590, seqNum=-1] 2024-12-09T06:17:58,065 DEBUG [Thread-2420 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T06:17:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T06:17:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:17:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:17:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:17:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:17:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:17:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5] 2024-12-09T06:17:58,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120949d064b53a95422eac5a32922da71cc1_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725078063/Put/seqid=0 2024-12-09T06:17:58,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742367_1543 (size=12154) 2024-12-09T06:17:58,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725078082, max timestamp=1733725078120Num uniques 859; 2024-12-09T06:17:58,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725078082, max timestamp=1733725078120Num uniques 859; 2024-12-09T06:17:58,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725078082, max timestamp=1733725078120Num uniques 859; 2024-12-09T06:17:58,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:58,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:58,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:17:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T06:17:58,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725138162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725138166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725138167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725138167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725138168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:58,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:17:58,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,221 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T06:17:58,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725138276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725138276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725138276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725138276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725138277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:58,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:17:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T06:17:58,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725138486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725138486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725138486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725138486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725138487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,500 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:58,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120949d064b53a95422eac5a32922da71cc1_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120949d064b53a95422eac5a32922da71cc1_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:58,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/89f96684cb3e4ec1812ccd79af5d1ab6, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:17:58,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/89f96684cb3e4ec1812ccd79af5d1ab6 is 175, key is test_row_0/A:col10/1733725078063/Put/seqid=0 2024-12-09T06:17:58,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742368_1544 (size=30955) 2024-12-09T06:17:58,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:17:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:58,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:17:58,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T06:17:58,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725138796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725138796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725138797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725138797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725138797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:58,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:17:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/89f96684cb3e4ec1812ccd79af5d1ab6 2024-12-09T06:17:58,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/44028456d11a4ab0a1f0c3393234ff79 is 50, key is test_row_0/B:col10/1733725078063/Put/seqid=0 2024-12-09T06:17:58,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742369_1545 (size=12001) 2024-12-09T06:17:58,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:17:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:58,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:59,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:17:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:59,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T06:17:59,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:17:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:59,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:17:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725139306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725139306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725139307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:59,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725139307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725139307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:17:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:17:59,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/44028456d11a4ab0a1f0c3393234ff79 2024-12-09T06:17:59,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/c0793431493940b69c71d924d0f53e43 is 50, key is test_row_0/C:col10/1733725078063/Put/seqid=0 2024-12-09T06:17:59,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742370_1546 (size=12001) 2024-12-09T06:17:59,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/c0793431493940b69c71d924d0f53e43 2024-12-09T06:17:59,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/89f96684cb3e4ec1812ccd79af5d1ab6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/89f96684cb3e4ec1812ccd79af5d1ab6 2024-12-09T06:17:59,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/89f96684cb3e4ec1812ccd79af5d1ab6, entries=150, sequenceid=15, filesize=30.2 K 2024-12-09T06:17:59,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/44028456d11a4ab0a1f0c3393234ff79 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/44028456d11a4ab0a1f0c3393234ff79 2024-12-09T06:17:59,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/44028456d11a4ab0a1f0c3393234ff79, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T06:17:59,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/c0793431493940b69c71d924d0f53e43 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c0793431493940b69c71d924d0f53e43 2024-12-09T06:17:59,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c0793431493940b69c71d924d0f53e43, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T06:17:59,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 86ed6927f9c6713f75d4b2e28d186f5a in 1360ms, sequenceid=15, compaction requested=false 2024-12-09T06:17:59,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:17:59,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T06:17:59,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:17:59,447 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T06:17:59,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:17:59,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:59,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:17:59,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:59,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:17:59,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:17:59,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091790476331134480821b41c90a6118a5_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725078167/Put/seqid=0 2024-12-09T06:17:59,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742371_1547 (size=12154) 2024-12-09T06:17:59,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:17:59,485 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091790476331134480821b41c90a6118a5_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091790476331134480821b41c90a6118a5_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:17:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/74f195c0a95f4dfa9418edccad2ac3dd, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:17:59,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/74f195c0a95f4dfa9418edccad2ac3dd is 175, key is test_row_0/A:col10/1733725078167/Put/seqid=0 2024-12-09T06:17:59,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742372_1548 (size=30955) 2024-12-09T06:17:59,494 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/74f195c0a95f4dfa9418edccad2ac3dd 2024-12-09T06:17:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/48c884183c4a492fb3f996f974214c1f is 50, key is test_row_0/B:col10/1733725078167/Put/seqid=0 2024-12-09T06:17:59,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742373_1549 (size=12001) 2024-12-09T06:17:59,910 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/48c884183c4a492fb3f996f974214c1f 2024-12-09T06:17:59,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/348f9bb1af4c4f9f9a3a9db7d7f909a7 is 50, key is test_row_0/C:col10/1733725078167/Put/seqid=0 2024-12-09T06:17:59,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742374_1550 (size=12001) 2024-12-09T06:18:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T06:18:00,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:00,325 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/348f9bb1af4c4f9f9a3a9db7d7f909a7 2024-12-09T06:18:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/74f195c0a95f4dfa9418edccad2ac3dd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/74f195c0a95f4dfa9418edccad2ac3dd 2024-12-09T06:18:00,334 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/74f195c0a95f4dfa9418edccad2ac3dd, entries=150, sequenceid=51, filesize=30.2 K 2024-12-09T06:18:00,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/48c884183c4a492fb3f996f974214c1f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/48c884183c4a492fb3f996f974214c1f 2024-12-09T06:18:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725140331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725140332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725140332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725140332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725140333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,341 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/48c884183c4a492fb3f996f974214c1f, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T06:18:00,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/348f9bb1af4c4f9f9a3a9db7d7f909a7 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/348f9bb1af4c4f9f9a3a9db7d7f909a7 2024-12-09T06:18:00,346 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/348f9bb1af4c4f9f9a3a9db7d7f909a7, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T06:18:00,347 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 86ed6927f9c6713f75d4b2e28d186f5a in 900ms, sequenceid=51, compaction requested=false 2024-12-09T06:18:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-09T06:18:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-09T06:18:00,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-09T06:18:00,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2840 sec 2024-12-09T06:18:00,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.2960 sec 2024-12-09T06:18:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:00,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T06:18:00,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:00,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:00,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:00,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:00,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:00,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:00,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120949f6eebabf604d62a66cc432c22a8ad3_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725080330/Put/seqid=0 2024-12-09T06:18:00,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742375_1551 (size=14594) 2024-12-09T06:18:00,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725140464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725140466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725140473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725140474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725140474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725140577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725140577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725140587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725140587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725140587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,737 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T06:18:00,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725140787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725140787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725140796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725140796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725140796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:00,873 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:00,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120949f6eebabf604d62a66cc432c22a8ad3_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120949f6eebabf604d62a66cc432c22a8ad3_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:00,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/421d023c96e94474a0d37cf0d5417ec3, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:00,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/421d023c96e94474a0d37cf0d5417ec3 is 175, key is test_row_0/A:col10/1733725080330/Put/seqid=0 2024-12-09T06:18:00,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742376_1552 (size=39549) 2024-12-09T06:18:00,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/421d023c96e94474a0d37cf0d5417ec3 2024-12-09T06:18:00,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/73dce4d23b7b498ea9c95b47fde26e72 is 50, key is test_row_0/B:col10/1733725080330/Put/seqid=0 2024-12-09T06:18:00,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742377_1553 (size=12001) 2024-12-09T06:18:01,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725141096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725141096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725141107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725141107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725141107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/73dce4d23b7b498ea9c95b47fde26e72 2024-12-09T06:18:01,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/79aa3634552642249e6cb530c5d2edba is 50, key is test_row_0/C:col10/1733725080330/Put/seqid=0 2024-12-09T06:18:01,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742378_1554 (size=12001) 2024-12-09T06:18:01,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/79aa3634552642249e6cb530c5d2edba 2024-12-09T06:18:01,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/421d023c96e94474a0d37cf0d5417ec3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/421d023c96e94474a0d37cf0d5417ec3 2024-12-09T06:18:01,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/421d023c96e94474a0d37cf0d5417ec3, entries=200, sequenceid=65, filesize=38.6 K 2024-12-09T06:18:01,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/73dce4d23b7b498ea9c95b47fde26e72 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/73dce4d23b7b498ea9c95b47fde26e72 2024-12-09T06:18:01,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/73dce4d23b7b498ea9c95b47fde26e72, entries=150, sequenceid=65, filesize=11.7 K 2024-12-09T06:18:01,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/79aa3634552642249e6cb530c5d2edba as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/79aa3634552642249e6cb530c5d2edba 2024-12-09T06:18:01,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/79aa3634552642249e6cb530c5d2edba, entries=150, sequenceid=65, filesize=11.7 K 2024-12-09T06:18:01,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 86ed6927f9c6713f75d4b2e28d186f5a in 937ms, sequenceid=65, compaction requested=true 2024-12-09T06:18:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:18:01,387 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:18:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:18:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:01,387 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:18:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:18:01,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:01,388 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:18:01,388 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:18:01,388 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/A is initiating minor compaction (all files) 2024-12-09T06:18:01,388 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/B is initiating minor compaction (all files) 2024-12-09T06:18:01,388 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/A in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:01,388 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/B in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:01,388 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/89f96684cb3e4ec1812ccd79af5d1ab6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/74f195c0a95f4dfa9418edccad2ac3dd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/421d023c96e94474a0d37cf0d5417ec3] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=99.1 K 2024-12-09T06:18:01,388 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/44028456d11a4ab0a1f0c3393234ff79, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/48c884183c4a492fb3f996f974214c1f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/73dce4d23b7b498ea9c95b47fde26e72] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=35.2 K 2024-12-09T06:18:01,388 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:01,388 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/89f96684cb3e4ec1812ccd79af5d1ab6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/74f195c0a95f4dfa9418edccad2ac3dd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/421d023c96e94474a0d37cf0d5417ec3] 2024-12-09T06:18:01,389 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 44028456d11a4ab0a1f0c3393234ff79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733725078063 2024-12-09T06:18:01,389 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 48c884183c4a492fb3f996f974214c1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733725078165 2024-12-09T06:18:01,389 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 73dce4d23b7b498ea9c95b47fde26e72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725080330 2024-12-09T06:18:01,389 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f96684cb3e4ec1812ccd79af5d1ab6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733725078063 2024-12-09T06:18:01,390 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74f195c0a95f4dfa9418edccad2ac3dd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733725078165 2024-12-09T06:18:01,390 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 421d023c96e94474a0d37cf0d5417ec3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725080330 2024-12-09T06:18:01,399 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:01,399 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/c8faa1b5d225426f95aad0b60bac5c33 is 50, key is test_row_0/B:col10/1733725080330/Put/seqid=0 2024-12-09T06:18:01,401 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:01,423 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120905c30c50006245dda9ff1a4d49d3cbcb_86ed6927f9c6713f75d4b2e28d186f5a store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:01,425 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120905c30c50006245dda9ff1a4d49d3cbcb_86ed6927f9c6713f75d4b2e28d186f5a, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:01,425 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120905c30c50006245dda9ff1a4d49d3cbcb_86ed6927f9c6713f75d4b2e28d186f5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:01,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742379_1555 (size=12104) 2024-12-09T06:18:01,444 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/c8faa1b5d225426f95aad0b60bac5c33 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c8faa1b5d225426f95aad0b60bac5c33 2024-12-09T06:18:01,449 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/B of 86ed6927f9c6713f75d4b2e28d186f5a into c8faa1b5d225426f95aad0b60bac5c33(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:01,449 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:01,449 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/B, priority=13, startTime=1733725081387; duration=0sec 2024-12-09T06:18:01,449 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:01,449 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:B 2024-12-09T06:18:01,449 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T06:18:01,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742380_1556 (size=4469) 2024-12-09T06:18:01,450 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T06:18:01,450 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/C is initiating minor compaction (all files) 2024-12-09T06:18:01,450 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/C in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:01,450 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c0793431493940b69c71d924d0f53e43, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/348f9bb1af4c4f9f9a3a9db7d7f909a7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/79aa3634552642249e6cb530c5d2edba] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=35.2 K 2024-12-09T06:18:01,451 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c0793431493940b69c71d924d0f53e43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733725078063 2024-12-09T06:18:01,451 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 348f9bb1af4c4f9f9a3a9db7d7f909a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733725078165 2024-12-09T06:18:01,451 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 79aa3634552642249e6cb530c5d2edba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725080330 2024-12-09T06:18:01,458 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#C#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:01,458 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/a5edf02cb8704befb6f7439b41a5a8c4 is 50, key is test_row_0/C:col10/1733725080330/Put/seqid=0 2024-12-09T06:18:01,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742381_1557 (size=12104) 2024-12-09T06:18:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:01,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T06:18:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:01,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095a4d717090de4525adee1df86379f0aa_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725081608/Put/seqid=0 2024-12-09T06:18:01,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742382_1558 (size=14594) 2024-12-09T06:18:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725141632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725141633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725141633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725141637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725141641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725141746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725141746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725141747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725141749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725141747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,850 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#A#compaction#478 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:01,851 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/d56edb9c8021426aa28a7946d59ea90f is 175, key is test_row_0/A:col10/1733725080330/Put/seqid=0 2024-12-09T06:18:01,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742383_1559 (size=31058) 2024-12-09T06:18:01,858 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/d56edb9c8021426aa28a7946d59ea90f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/d56edb9c8021426aa28a7946d59ea90f 2024-12-09T06:18:01,861 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/A of 86ed6927f9c6713f75d4b2e28d186f5a into d56edb9c8021426aa28a7946d59ea90f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:01,861 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:01,861 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/A, priority=13, startTime=1733725081387; duration=0sec 2024-12-09T06:18:01,861 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:01,861 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:A 2024-12-09T06:18:01,870 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/a5edf02cb8704befb6f7439b41a5a8c4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/a5edf02cb8704befb6f7439b41a5a8c4 2024-12-09T06:18:01,874 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/C of 86ed6927f9c6713f75d4b2e28d186f5a into a5edf02cb8704befb6f7439b41a5a8c4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:01,874 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:01,874 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/C, priority=13, startTime=1733725081387; duration=0sec 2024-12-09T06:18:01,875 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:01,875 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:C 2024-12-09T06:18:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725141956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725141957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725141957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725141966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725141966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,029 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:02,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095a4d717090de4525adee1df86379f0aa_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095a4d717090de4525adee1df86379f0aa_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:02,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/b616fbc700984234ae6206be1e9b3267, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:02,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/b616fbc700984234ae6206be1e9b3267 is 175, key is test_row_0/A:col10/1733725081608/Put/seqid=0 2024-12-09T06:18:02,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742384_1560 (size=39549) 2024-12-09T06:18:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T06:18:02,217 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:02,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:18:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-09T06:18:02,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:18:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T06:18:02,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:18:02,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:18:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725142267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725142267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725142267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725142277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725142277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T06:18:02,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T06:18:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:02,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:02,441 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/b616fbc700984234ae6206be1e9b3267 2024-12-09T06:18:02,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/672a3cd0dccf4f12b8e35f9270b3bd07 is 50, key is test_row_0/B:col10/1733725081608/Put/seqid=0 2024-12-09T06:18:02,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742385_1561 (size=12001) 2024-12-09T06:18:02,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/672a3cd0dccf4f12b8e35f9270b3bd07 2024-12-09T06:18:02,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/e2c912859a2f49009e42f21a15947542 is 50, key is test_row_0/C:col10/1733725081608/Put/seqid=0 2024-12-09T06:18:02,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742386_1562 (size=12001) 2024-12-09T06:18:02,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/e2c912859a2f49009e42f21a15947542 2024-12-09T06:18:02,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/b616fbc700984234ae6206be1e9b3267 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/b616fbc700984234ae6206be1e9b3267 2024-12-09T06:18:02,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/b616fbc700984234ae6206be1e9b3267, entries=200, sequenceid=86, filesize=38.6 K 2024-12-09T06:18:02,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/672a3cd0dccf4f12b8e35f9270b3bd07 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/672a3cd0dccf4f12b8e35f9270b3bd07 2024-12-09T06:18:02,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/672a3cd0dccf4f12b8e35f9270b3bd07, entries=150, sequenceid=86, filesize=11.7 K 2024-12-09T06:18:02,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/e2c912859a2f49009e42f21a15947542 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e2c912859a2f49009e42f21a15947542 2024-12-09T06:18:02,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e2c912859a2f49009e42f21a15947542, entries=150, sequenceid=86, filesize=11.7 K 2024-12-09T06:18:02,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 86ed6927f9c6713f75d4b2e28d186f5a in 859ms, sequenceid=86, compaction requested=false 2024-12-09T06:18:02,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:02,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T06:18:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:02,525 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T06:18:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:02,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:02,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:02,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209072f41d95d704d288c1cd22fc1c335ee_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725081634/Put/seqid=0 2024-12-09T06:18:02,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742387_1563 (size=12154) 2024-12-09T06:18:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T06:18:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:02,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:02,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725142792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725142795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725142796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725142797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725142800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T06:18:02,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725142907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725142907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725142907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725142907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725142916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:02,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:02,936 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209072f41d95d704d288c1cd22fc1c335ee_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209072f41d95d704d288c1cd22fc1c335ee_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/aa642c0fbd4944c1be747ea18d4c8861, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:02,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/aa642c0fbd4944c1be747ea18d4c8861 is 175, key is test_row_0/A:col10/1733725081634/Put/seqid=0 2024-12-09T06:18:02,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742388_1564 (size=30955) 2024-12-09T06:18:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725143116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725143116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725143116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725143116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725143126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,340 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/aa642c0fbd4944c1be747ea18d4c8861 2024-12-09T06:18:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/8bce83d7ffb140cdbbfc0c9b6530f4b0 is 50, key is test_row_0/B:col10/1733725081634/Put/seqid=0 2024-12-09T06:18:03,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742389_1565 (size=12001) 2024-12-09T06:18:03,349 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/8bce83d7ffb140cdbbfc0c9b6530f4b0 2024-12-09T06:18:03,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/41c4fa8cc9ef41239923958200c32f13 is 50, key is test_row_0/C:col10/1733725081634/Put/seqid=0 2024-12-09T06:18:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T06:18:03,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742390_1566 (size=12001) 2024-12-09T06:18:03,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725143427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725143427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725143427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725143427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725143436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:03,757 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/41c4fa8cc9ef41239923958200c32f13 2024-12-09T06:18:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/aa642c0fbd4944c1be747ea18d4c8861 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/aa642c0fbd4944c1be747ea18d4c8861 2024-12-09T06:18:03,763 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/aa642c0fbd4944c1be747ea18d4c8861, entries=150, sequenceid=100, filesize=30.2 K 2024-12-09T06:18:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/8bce83d7ffb140cdbbfc0c9b6530f4b0 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/8bce83d7ffb140cdbbfc0c9b6530f4b0 2024-12-09T06:18:03,766 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/8bce83d7ffb140cdbbfc0c9b6530f4b0, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T06:18:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/41c4fa8cc9ef41239923958200c32f13 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/41c4fa8cc9ef41239923958200c32f13 2024-12-09T06:18:03,770 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/41c4fa8cc9ef41239923958200c32f13, entries=150, sequenceid=100, filesize=11.7 K 2024-12-09T06:18:03,771 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 86ed6927f9c6713f75d4b2e28d186f5a in 1246ms, sequenceid=100, compaction requested=true 2024-12-09T06:18:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:03,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-09T06:18:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-09T06:18:03,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-09T06:18:03,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5530 sec 2024-12-09T06:18:03,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.5570 sec 2024-12-09T06:18:03,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725082794, max timestamp=1733725083938Num uniques 859; 2024-12-09T06:18:03,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:03,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725082794, max timestamp=1733725083938Num uniques 859; 2024-12-09T06:18:03,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:03,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-09T06:18:03,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:03,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:03,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:03,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:03,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:03,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:03,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096bed56de1d224939a6022b3a166d24a4_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725082796/Put/seqid=0 2024-12-09T06:18:03,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742391_1567 (size=12154) 2024-12-09T06:18:03,975 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:03,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733725083967, max timestamp=1733725083980Num uniques 859; 2024-12-09T06:18:03,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:18:03,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733725083967, max timestamp=1733725083980Num uniques 859; 2024-12-09T06:18:03,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:18:03,995 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096bed56de1d224939a6022b3a166d24a4_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096bed56de1d224939a6022b3a166d24a4_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:03,995 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/fd89fc99072943e19e90f2da2766fadf, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:03,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/fd89fc99072943e19e90f2da2766fadf is 175, key is test_row_0/A:col10/1733725082796/Put/seqid=0 2024-12-09T06:18:04,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742392_1568 (size=30955) 2024-12-09T06:18:04,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725144019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725144020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725144020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725144021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725144026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725144138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725144138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725144138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725144138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725144138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725144347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725144347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725144347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725144347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725144356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T06:18:04,367 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:04,367 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:18:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-09T06:18:04,369 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:18:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T06:18:04,370 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:18:04,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:18:04,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/fd89fc99072943e19e90f2da2766fadf 2024-12-09T06:18:04,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/dc6c3fe4841747f183c433538d598e13 is 50, key is test_row_0/B:col10/1733725082796/Put/seqid=0 2024-12-09T06:18:04,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742393_1569 (size=12001) 2024-12-09T06:18:04,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/dc6c3fe4841747f183c433538d598e13 2024-12-09T06:18:04,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/c9c4d6df03f94989a04dd01114c77a29 is 50, key is test_row_0/C:col10/1733725082796/Put/seqid=0 2024-12-09T06:18:04,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742394_1570 (size=12001) 2024-12-09T06:18:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T06:18:04,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T06:18:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725144656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725144656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725144657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725144657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725144666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:04,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T06:18:04,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T06:18:04,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T06:18:04,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:04,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:04,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/c9c4d6df03f94989a04dd01114c77a29 2024-12-09T06:18:04,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/fd89fc99072943e19e90f2da2766fadf as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fd89fc99072943e19e90f2da2766fadf 2024-12-09T06:18:04,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fd89fc99072943e19e90f2da2766fadf, entries=150, sequenceid=122, filesize=30.2 K 2024-12-09T06:18:04,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/dc6c3fe4841747f183c433538d598e13 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/dc6c3fe4841747f183c433538d598e13 2024-12-09T06:18:04,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/dc6c3fe4841747f183c433538d598e13, entries=150, sequenceid=122, filesize=11.7 K 2024-12-09T06:18:04,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/c9c4d6df03f94989a04dd01114c77a29 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c9c4d6df03f94989a04dd01114c77a29 2024-12-09T06:18:04,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c9c4d6df03f94989a04dd01114c77a29, entries=150, sequenceid=122, filesize=11.7 K 2024-12-09T06:18:04,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=116.29 KB/119080 for 86ed6927f9c6713f75d4b2e28d186f5a in 927ms, sequenceid=122, compaction requested=true 2024-12-09T06:18:04,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:04,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:18:04,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:04,869 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:04,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:18:04,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:04,869 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:04,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:18:04,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:04,869 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:04,869 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:04,869 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/B is initiating minor compaction (all files) 2024-12-09T06:18:04,869 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/A is initiating minor compaction (all files) 2024-12-09T06:18:04,870 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/A in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,870 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/B in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,870 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/d56edb9c8021426aa28a7946d59ea90f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/b616fbc700984234ae6206be1e9b3267, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/aa642c0fbd4944c1be747ea18d4c8861, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fd89fc99072943e19e90f2da2766fadf] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=129.4 K 2024-12-09T06:18:04,870 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c8faa1b5d225426f95aad0b60bac5c33, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/672a3cd0dccf4f12b8e35f9270b3bd07, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/8bce83d7ffb140cdbbfc0c9b6530f4b0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/dc6c3fe4841747f183c433538d598e13] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=47.0 K 2024-12-09T06:18:04,870 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,870 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/d56edb9c8021426aa28a7946d59ea90f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/b616fbc700984234ae6206be1e9b3267, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/aa642c0fbd4944c1be747ea18d4c8861, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fd89fc99072943e19e90f2da2766fadf] 2024-12-09T06:18:04,870 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c8faa1b5d225426f95aad0b60bac5c33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725080330 2024-12-09T06:18:04,870 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting d56edb9c8021426aa28a7946d59ea90f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725080330 2024-12-09T06:18:04,870 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 672a3cd0dccf4f12b8e35f9270b3bd07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733725080463 2024-12-09T06:18:04,870 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting b616fbc700984234ae6206be1e9b3267, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733725080463 2024-12-09T06:18:04,870 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bce83d7ffb140cdbbfc0c9b6530f4b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733725081616 2024-12-09T06:18:04,870 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa642c0fbd4944c1be747ea18d4c8861, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733725081616 2024-12-09T06:18:04,871 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting dc6c3fe4841747f183c433538d598e13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733725082796 2024-12-09T06:18:04,871 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd89fc99072943e19e90f2da2766fadf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733725082796 2024-12-09T06:18:04,876 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:04,877 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209b605f11e66104ecbb22132cb8bc5ba21_86ed6927f9c6713f75d4b2e28d186f5a store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:04,879 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#B#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:04,879 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209b605f11e66104ecbb22132cb8bc5ba21_86ed6927f9c6713f75d4b2e28d186f5a, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:04,879 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b605f11e66104ecbb22132cb8bc5ba21_86ed6927f9c6713f75d4b2e28d186f5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:04,879 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/5e953cfbd4d04c12a18ee52e39c569fc is 50, key is test_row_0/B:col10/1733725082796/Put/seqid=0 2024-12-09T06:18:04,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742395_1571 (size=4469) 2024-12-09T06:18:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742396_1572 (size=12241) 2024-12-09T06:18:04,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T06:18:04,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:04,984 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=116.29 KB heapSize=333.70 KB 2024-12-09T06:18:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209507dd3cd3e8f401d9d9b75988ac71ebc_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725084019/Put/seqid=0 2024-12-09T06:18:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T06:18:05,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742397_1573 (size=12304) 2024-12-09T06:18:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:05,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:05,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725145176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725145176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725145178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725145178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725145179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,283 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#A#compaction#489 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:05,284 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/2656a60949084f278fc1bfc98a3795de is 175, key is test_row_0/A:col10/1733725082796/Put/seqid=0 2024-12-09T06:18:05,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742398_1574 (size=31195) 2024-12-09T06:18:05,289 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/5e953cfbd4d04c12a18ee52e39c569fc as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/5e953cfbd4d04c12a18ee52e39c569fc 2024-12-09T06:18:05,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725145286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,291 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/B of 86ed6927f9c6713f75d4b2e28d186f5a into 5e953cfbd4d04c12a18ee52e39c569fc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:05,291 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:05,292 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/B, priority=12, startTime=1733725084869; duration=0sec 2024-12-09T06:18:05,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,292 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:05,292 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:B 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725145286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,292 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725145286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,292 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:05,293 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/C is initiating minor compaction (all files) 2024-12-09T06:18:05,293 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/C in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:05,293 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/a5edf02cb8704befb6f7439b41a5a8c4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e2c912859a2f49009e42f21a15947542, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/41c4fa8cc9ef41239923958200c32f13, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c9c4d6df03f94989a04dd01114c77a29] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=47.0 K 2024-12-09T06:18:05,293 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting a5edf02cb8704befb6f7439b41a5a8c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733725080330 2024-12-09T06:18:05,293 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e2c912859a2f49009e42f21a15947542, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733725080463 2024-12-09T06:18:05,293 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c4fa8cc9ef41239923958200c32f13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733725081616 2024-12-09T06:18:05,293 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c9c4d6df03f94989a04dd01114c77a29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733725082796 2024-12-09T06:18:05,298 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:05,298 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/24bf6498a02e47d79c04696b67cd8a43 is 50, key is test_row_0/C:col10/1733725082796/Put/seqid=0 2024-12-09T06:18:05,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742399_1575 (size=12241) 2024-12-09T06:18:05,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:05,439 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209507dd3cd3e8f401d9d9b75988ac71ebc_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209507dd3cd3e8f401d9d9b75988ac71ebc_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/4ef52e15e38547a9a22a2760c7f33594, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/4ef52e15e38547a9a22a2760c7f33594 is 175, key is test_row_0/A:col10/1733725084019/Put/seqid=0 2024-12-09T06:18:05,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742400_1576 (size=31105) 2024-12-09T06:18:05,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725145497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725145497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725145497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T06:18:05,689 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/2656a60949084f278fc1bfc98a3795de as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/2656a60949084f278fc1bfc98a3795de 2024-12-09T06:18:05,693 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/A of 86ed6927f9c6713f75d4b2e28d186f5a into 2656a60949084f278fc1bfc98a3795de(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:05,693 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:05,693 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/A, priority=12, startTime=1733725084869; duration=0sec 2024-12-09T06:18:05,693 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:05,693 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:A 2024-12-09T06:18:05,707 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/24bf6498a02e47d79c04696b67cd8a43 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/24bf6498a02e47d79c04696b67cd8a43 2024-12-09T06:18:05,710 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/C of 86ed6927f9c6713f75d4b2e28d186f5a into 24bf6498a02e47d79c04696b67cd8a43(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:05,710 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:05,711 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/C, priority=12, startTime=1733725084869; duration=0sec 2024-12-09T06:18:05,711 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:05,711 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:C 2024-12-09T06:18:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725145807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725145807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725145807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:05,863 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/4ef52e15e38547a9a22a2760c7f33594 2024-12-09T06:18:05,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/185ecfd5c6b94f499f9e3b0e77504224 is 50, key is test_row_0/B:col10/1733725084019/Put/seqid=0 2024-12-09T06:18:05,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742401_1577 (size=12151) 2024-12-09T06:18:06,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725146186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:06,188 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:06,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725146197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:06,200 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:06,307 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/185ecfd5c6b94f499f9e3b0e77504224 2024-12-09T06:18:06,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/3f1feee252b44f7da43c92501e23b6da is 50, key is test_row_0/C:col10/1733725084019/Put/seqid=0 2024-12-09T06:18:06,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725146317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725146317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725146317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:06,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742402_1578 (size=12151) 2024-12-09T06:18:06,332 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/3f1feee252b44f7da43c92501e23b6da 2024-12-09T06:18:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/4ef52e15e38547a9a22a2760c7f33594 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4ef52e15e38547a9a22a2760c7f33594 2024-12-09T06:18:06,344 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4ef52e15e38547a9a22a2760c7f33594, entries=150, sequenceid=151, filesize=30.4 K 2024-12-09T06:18:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/185ecfd5c6b94f499f9e3b0e77504224 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/185ecfd5c6b94f499f9e3b0e77504224 2024-12-09T06:18:06,359 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/185ecfd5c6b94f499f9e3b0e77504224, entries=150, sequenceid=151, filesize=11.9 K 2024-12-09T06:18:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/3f1feee252b44f7da43c92501e23b6da as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3f1feee252b44f7da43c92501e23b6da 2024-12-09T06:18:06,365 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3f1feee252b44f7da43c92501e23b6da, entries=150, sequenceid=151, filesize=11.9 K 2024-12-09T06:18:06,366 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~116.29 KB/119080, heapSize ~333.66 KB/341664, currentSize=60.38 KB/61830 for 86ed6927f9c6713f75d4b2e28d186f5a in 1383ms, sequenceid=151, compaction requested=false 2024-12-09T06:18:06,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:06,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:06,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-09T06:18:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-09T06:18:06,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-09T06:18:06,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9980 sec 2024-12-09T06:18:06,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.0040 sec 2024-12-09T06:18:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T06:18:06,527 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:06,527 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:18:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-09T06:18:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T06:18:06,530 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:18:06,530 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:18:06,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:18:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T06:18:06,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T06:18:06,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:06,685 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T06:18:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:06,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f97e0d60f9d549319ff0d9b86a77e0a2_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725085177/Put/seqid=0 2024-12-09T06:18:06,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742403_1579 (size=12304) 2024-12-09T06:18:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:06,708 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f97e0d60f9d549319ff0d9b86a77e0a2_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f97e0d60f9d549319ff0d9b86a77e0a2_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:06,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6a6f6ef3306d4e5fa9bdccdc90170cb4, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:06,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6a6f6ef3306d4e5fa9bdccdc90170cb4 is 175, key is test_row_0/A:col10/1733725085177/Put/seqid=0 2024-12-09T06:18:06,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742404_1580 (size=31105) 2024-12-09T06:18:06,716 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6a6f6ef3306d4e5fa9bdccdc90170cb4 2024-12-09T06:18:06,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/4270d0a00f3c40a982c5848f034fd3cf is 50, key is test_row_0/B:col10/1733725085177/Put/seqid=0 2024-12-09T06:18:06,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742405_1581 (size=12151) 2024-12-09T06:18:06,731 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/4270d0a00f3c40a982c5848f034fd3cf 2024-12-09T06:18:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/8696176589794aa9b7334c0ff78d4caf is 50, key is test_row_0/C:col10/1733725085177/Put/seqid=0 2024-12-09T06:18:06,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742406_1582 (size=12151) 2024-12-09T06:18:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T06:18:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T06:18:07,160 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/8696176589794aa9b7334c0ff78d4caf 2024-12-09T06:18:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6a6f6ef3306d4e5fa9bdccdc90170cb4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6a6f6ef3306d4e5fa9bdccdc90170cb4 2024-12-09T06:18:07,165 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6a6f6ef3306d4e5fa9bdccdc90170cb4, entries=150, sequenceid=166, filesize=30.4 K 2024-12-09T06:18:07,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/4270d0a00f3c40a982c5848f034fd3cf as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4270d0a00f3c40a982c5848f034fd3cf 2024-12-09T06:18:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,168 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4270d0a00f3c40a982c5848f034fd3cf, entries=150, sequenceid=166, filesize=11.9 K 2024-12-09T06:18:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/8696176589794aa9b7334c0ff78d4caf as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8696176589794aa9b7334c0ff78d4caf 2024-12-09T06:18:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,171 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8696176589794aa9b7334c0ff78d4caf, entries=150, sequenceid=166, filesize=11.9 K 2024-12-09T06:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,172 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 86ed6927f9c6713f75d4b2e28d186f5a in 489ms, sequenceid=166, compaction requested=true 2024-12-09T06:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-09T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-09T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-09T06:18:07,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 642 msec 2024-12-09T06:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 647 msec 2024-12-09T06:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:07,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T06:18:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:07,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:07,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:07,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:07,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:07,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209bbf48380045b4c52b7560bb54d4cd061_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725087360/Put/seqid=0 2024-12-09T06:18:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742407_1583 (size=19774) 2024-12-09T06:18:07,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725087395, max timestamp=1733725087407Num uniques 859; 2024-12-09T06:18:07,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:07,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725087395, max timestamp=1733725087407Num uniques 859; 2024-12-09T06:18:07,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:07,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725087395, max timestamp=1733725087407Num uniques 859; 2024-12-09T06:18:07,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:07,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725087433, max timestamp=1733725087456Num uniques 1018; 2024-12-09T06:18:07,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:18:07,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725087433, max timestamp=1733725087456Num uniques 1018; 2024-12-09T06:18:07,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:18:07,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733725087433, max timestamp=1733725087456Num uniques 1018; 2024-12-09T06:18:07,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T06:18:07,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725147530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725147536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725147554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725147647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725147656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T06:18:07,666 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:07,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:18:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-09T06:18:07,668 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:18:07,669 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:18:07,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:18:07,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725147666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T06:18:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T06:18:07,784 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:07,787 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209bbf48380045b4c52b7560bb54d4cd061_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bbf48380045b4c52b7560bb54d4cd061_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:07,788 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/0623ee7dae4f4e5abd891be796b6c41e, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:07,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/0623ee7dae4f4e5abd891be796b6c41e is 175, key is test_row_0/A:col10/1733725087360/Put/seqid=0 2024-12-09T06:18:07,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742408_1584 (size=57033) 2024-12-09T06:18:07,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:07,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:07,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:07,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:07,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:07,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:07,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725147857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725147866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725147876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:07,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:07,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:07,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:07,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:07,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:07,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T06:18:08,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:08,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:08,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725148166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725148177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725148186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,193 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/0623ee7dae4f4e5abd891be796b6c41e 2024-12-09T06:18:08,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/2e33549592484829bcbc20ba87df9894 is 50, key is test_row_0/B:col10/1733725087360/Put/seqid=0 2024-12-09T06:18:08,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742409_1585 (size=12151) 2024-12-09T06:18:08,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/2e33549592484829bcbc20ba87df9894 2024-12-09T06:18:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725148206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,209 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/9176c98882ee4f68821060c71cb90abd is 50, key is test_row_0/C:col10/1733725087360/Put/seqid=0 2024-12-09T06:18:08,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742410_1586 (size=12151) 2024-12-09T06:18:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725148216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,219 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:08,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:08,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T06:18:08,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:08,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:08,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/9176c98882ee4f68821060c71cb90abd 2024-12-09T06:18:08,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/0623ee7dae4f4e5abd891be796b6c41e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/0623ee7dae4f4e5abd891be796b6c41e 2024-12-09T06:18:08,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/0623ee7dae4f4e5abd891be796b6c41e, entries=300, sequenceid=176, filesize=55.7 K 2024-12-09T06:18:08,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/2e33549592484829bcbc20ba87df9894 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/2e33549592484829bcbc20ba87df9894 2024-12-09T06:18:08,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/2e33549592484829bcbc20ba87df9894, entries=150, sequenceid=176, filesize=11.9 K 2024-12-09T06:18:08,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/9176c98882ee4f68821060c71cb90abd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/9176c98882ee4f68821060c71cb90abd 2024-12-09T06:18:08,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/9176c98882ee4f68821060c71cb90abd, entries=150, sequenceid=176, filesize=11.9 K 2024-12-09T06:18:08,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 86ed6927f9c6713f75d4b2e28d186f5a in 1258ms, sequenceid=176, compaction requested=true 2024-12-09T06:18:08,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:08,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:18:08,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:08,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:18:08,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:18:08,627 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:08,627 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:08,627 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:08,627 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:08,627 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/A is initiating minor compaction (all files) 2024-12-09T06:18:08,627 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/B is initiating minor compaction (all files) 2024-12-09T06:18:08,627 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/B in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,627 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/A in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,627 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/5e953cfbd4d04c12a18ee52e39c569fc, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/185ecfd5c6b94f499f9e3b0e77504224, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4270d0a00f3c40a982c5848f034fd3cf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/2e33549592484829bcbc20ba87df9894] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=47.6 K 2024-12-09T06:18:08,627 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/2656a60949084f278fc1bfc98a3795de, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4ef52e15e38547a9a22a2760c7f33594, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6a6f6ef3306d4e5fa9bdccdc90170cb4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/0623ee7dae4f4e5abd891be796b6c41e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=146.9 K 2024-12-09T06:18:08,627 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,627 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/2656a60949084f278fc1bfc98a3795de, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4ef52e15e38547a9a22a2760c7f33594, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6a6f6ef3306d4e5fa9bdccdc90170cb4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/0623ee7dae4f4e5abd891be796b6c41e] 2024-12-09T06:18:08,628 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e953cfbd4d04c12a18ee52e39c569fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733725082796 2024-12-09T06:18:08,628 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2656a60949084f278fc1bfc98a3795de, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733725082796 2024-12-09T06:18:08,628 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 185ecfd5c6b94f499f9e3b0e77504224, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733725083980 2024-12-09T06:18:08,628 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4270d0a00f3c40a982c5848f034fd3cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733725085177 2024-12-09T06:18:08,628 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef52e15e38547a9a22a2760c7f33594, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733725083980 2024-12-09T06:18:08,628 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e33549592484829bcbc20ba87df9894, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733725087353 2024-12-09T06:18:08,628 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a6f6ef3306d4e5fa9bdccdc90170cb4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733725085177 2024-12-09T06:18:08,628 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0623ee7dae4f4e5abd891be796b6c41e, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733725087353 2024-12-09T06:18:08,633 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:08,634 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:08,635 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/ce2cc94b275a4e34b32c4f5cd641e88f is 50, key is test_row_0/B:col10/1733725087360/Put/seqid=0 2024-12-09T06:18:08,636 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412097ef6f9449b544cc9930b517b8fc8e555_86ed6927f9c6713f75d4b2e28d186f5a store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:08,638 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412097ef6f9449b544cc9930b517b8fc8e555_86ed6927f9c6713f75d4b2e28d186f5a, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:08,638 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097ef6f9449b544cc9930b517b8fc8e555_86ed6927f9c6713f75d4b2e28d186f5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:08,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742411_1587 (size=12527) 2024-12-09T06:18:08,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742412_1588 (size=4469) 2024-12-09T06:18:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:08,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-09T06:18:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:18:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:18:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T06:18:08,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209075dd9cd57184b10ac0b9b2cfd4e027f_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725087501/Put/seqid=0 2024-12-09T06:18:08,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742413_1589 (size=12304) 2024-12-09T06:18:08,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725148691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725148696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725148696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T06:18:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725148806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725148806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:08,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:08,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:08,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725149016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725149016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:09,042 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/ce2cc94b275a4e34b32c4f5cd641e88f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/ce2cc94b275a4e34b32c4f5cd641e88f 2024-12-09T06:18:09,043 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#A#compaction#502 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:09,043 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6d4fb84b4c6e40609c8df63a450e35f3 is 175, key is test_row_0/A:col10/1733725087360/Put/seqid=0 2024-12-09T06:18:09,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742414_1590 (size=31481) 2024-12-09T06:18:09,051 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/B of 86ed6927f9c6713f75d4b2e28d186f5a into ce2cc94b275a4e34b32c4f5cd641e88f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:09,051 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:09,051 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/B, priority=12, startTime=1733725088626; duration=0sec 2024-12-09T06:18:09,051 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:09,051 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:B 2024-12-09T06:18:09,051 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:09,052 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6d4fb84b4c6e40609c8df63a450e35f3 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6d4fb84b4c6e40609c8df63a450e35f3 2024-12-09T06:18:09,052 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:09,052 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/C is initiating minor compaction (all files) 2024-12-09T06:18:09,053 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/C in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,053 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/24bf6498a02e47d79c04696b67cd8a43, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3f1feee252b44f7da43c92501e23b6da, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8696176589794aa9b7334c0ff78d4caf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/9176c98882ee4f68821060c71cb90abd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=47.6 K 2024-12-09T06:18:09,053 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 24bf6498a02e47d79c04696b67cd8a43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733725082796 2024-12-09T06:18:09,053 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f1feee252b44f7da43c92501e23b6da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733725083980 2024-12-09T06:18:09,053 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8696176589794aa9b7334c0ff78d4caf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733725085177 2024-12-09T06:18:09,054 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9176c98882ee4f68821060c71cb90abd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733725087353 2024-12-09T06:18:09,056 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/A of 86ed6927f9c6713f75d4b2e28d186f5a into 6d4fb84b4c6e40609c8df63a450e35f3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:09,057 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:09,057 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/A, priority=12, startTime=1733725088626; duration=0sec 2024-12-09T06:18:09,057 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:09,057 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:A 2024-12-09T06:18:09,060 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:09,061 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/877c5d171d2046c9aec59f7b30e75f69 is 50, key is test_row_0/C:col10/1733725087360/Put/seqid=0 2024-12-09T06:18:09,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742415_1591 (size=12527) 2024-12-09T06:18:09,070 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/877c5d171d2046c9aec59f7b30e75f69 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/877c5d171d2046c9aec59f7b30e75f69 2024-12-09T06:18:09,074 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/C of 86ed6927f9c6713f75d4b2e28d186f5a into 877c5d171d2046c9aec59f7b30e75f69(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:09,074 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:09,074 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/C, priority=12, startTime=1733725088626; duration=0sec 2024-12-09T06:18:09,074 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:09,074 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:C 2024-12-09T06:18:09,087 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:09,090 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209075dd9cd57184b10ac0b9b2cfd4e027f_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209075dd9cd57184b10ac0b9b2cfd4e027f_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:09,090 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/960ac0de74a14790b519da50f1bd47cd, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:09,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/960ac0de74a14790b519da50f1bd47cd is 175, key is test_row_0/A:col10/1733725087501/Put/seqid=0 2024-12-09T06:18:09,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742416_1592 (size=31105) 2024-12-09T06:18:09,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/960ac0de74a14790b519da50f1bd47cd 2024-12-09T06:18:09,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/b4cb39d8e3814b20b7de41f316e0eaac is 50, key is test_row_0/B:col10/1733725087501/Put/seqid=0 2024-12-09T06:18:09,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742417_1593 (size=12151) 2024-12-09T06:18:09,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725149327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725149327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:09,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:09,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:09,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/b4cb39d8e3814b20b7de41f316e0eaac 2024-12-09T06:18:09,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/4cd38a059dd04756a8e7a764b637962f is 50, key is test_row_0/C:col10/1733725087501/Put/seqid=0 2024-12-09T06:18:09,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742418_1594 (size=12151) 2024-12-09T06:18:09,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725149717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:09,718 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:09,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:09,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:09,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T06:18:09,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:09,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725149837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725149837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:09,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/4cd38a059dd04756a8e7a764b637962f 2024-12-09T06:18:09,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/960ac0de74a14790b519da50f1bd47cd as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/960ac0de74a14790b519da50f1bd47cd 2024-12-09T06:18:09,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/960ac0de74a14790b519da50f1bd47cd, entries=150, sequenceid=231, filesize=30.4 K 2024-12-09T06:18:09,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/b4cb39d8e3814b20b7de41f316e0eaac as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/b4cb39d8e3814b20b7de41f316e0eaac 2024-12-09T06:18:09,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/b4cb39d8e3814b20b7de41f316e0eaac, entries=150, sequenceid=231, filesize=11.9 K 2024-12-09T06:18:09,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/4cd38a059dd04756a8e7a764b637962f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/4cd38a059dd04756a8e7a764b637962f 2024-12-09T06:18:09,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/4cd38a059dd04756a8e7a764b637962f, entries=150, sequenceid=231, filesize=11.9 K 2024-12-09T06:18:09,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 86ed6927f9c6713f75d4b2e28d186f5a in 1254ms, sequenceid=231, compaction requested=false 2024-12-09T06:18:09,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:09,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T06:18:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:09,959 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T06:18:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:09,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:09,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f54ef36d82014531b035fd40bbc9109a_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725088685/Put/seqid=0 2024-12-09T06:18:09,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742419_1595 (size=12304) 2024-12-09T06:18:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:09,971 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f54ef36d82014531b035fd40bbc9109a_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f54ef36d82014531b035fd40bbc9109a_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:09,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/21f742a64b5f4febaba208090a36151e, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:09,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/21f742a64b5f4febaba208090a36151e is 175, key is test_row_0/A:col10/1733725088685/Put/seqid=0 2024-12-09T06:18:09,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742420_1596 (size=31105) 2024-12-09T06:18:10,376 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/21f742a64b5f4febaba208090a36151e 2024-12-09T06:18:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/c4d133afc9dd48c8a679bb10a85ec4a8 is 50, key is test_row_0/B:col10/1733725088685/Put/seqid=0 2024-12-09T06:18:10,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742421_1597 (size=12151) 2024-12-09T06:18:10,784 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/c4d133afc9dd48c8a679bb10a85ec4a8 2024-12-09T06:18:10,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/74d37ef72a964b1bae9c3edd768cf54d is 50, key is test_row_0/C:col10/1733725088685/Put/seqid=0 2024-12-09T06:18:10,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742422_1598 (size=12151) 2024-12-09T06:18:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:10,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:10,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725090877, max timestamp=1733725090887Num uniques 859; 2024-12-09T06:18:10,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:10,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725090877, max timestamp=1733725090887Num uniques 859; 2024-12-09T06:18:10,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:10,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725090877, max timestamp=1733725090887Num uniques 859; 2024-12-09T06:18:10,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:10,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733725090911, max timestamp=1733725090928Num uniques 1018; 2024-12-09T06:18:10,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:18:10,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733725090911, max timestamp=1733725090928Num uniques 1018; 2024-12-09T06:18:10,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:18:10,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733725090911, max timestamp=1733725090928Num uniques 1018; 2024-12-09T06:18:10,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:18:10,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733725090946, max timestamp=1733725090963Num uniques 1027; 2024-12-09T06:18:10,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733725090946, max timestamp=1733725090963Num uniques 1027; 2024-12-09T06:18:10,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733725090946, max timestamp=1733725090963Num uniques 1027; 2024-12-09T06:18:10,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T06:18:10,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T06:18:10,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T06:18:11,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725151014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733725091002, max timestamp=1733725091016Num uniques 1086; 2024-12-09T06:18:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T06:18:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733725091002, max timestamp=1733725091016Num uniques 1086; 2024-12-09T06:18:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733725091002, max timestamp=1733725091016Num uniques 1086; 2024-12-09T06:18:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T06:18:11,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-09T06:18:11,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733725091069, max timestamp=1733725091085Num uniques 1045; 2024-12-09T06:18:11,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733725091069, max timestamp=1733725091085Num uniques 1045; 2024-12-09T06:18:11,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T06:18:11,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T06:18:11,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733725091069, max timestamp=1733725091085Num uniques 1045; 2024-12-09T06:18:11,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T06:18:11,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725151144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733725091136, max timestamp=1733725091147Num uniques 1054; 2024-12-09T06:18:11,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733725091136, max timestamp=1733725091147Num uniques 1054; 2024-12-09T06:18:11,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T06:18:11,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T06:18:11,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733725091136, max timestamp=1733725091147Num uniques 1054; 2024-12-09T06:18:11,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T06:18:11,192 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/74d37ef72a964b1bae9c3edd768cf54d 2024-12-09T06:18:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/21f742a64b5f4febaba208090a36151e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/21f742a64b5f4febaba208090a36151e 2024-12-09T06:18:11,198 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/21f742a64b5f4febaba208090a36151e, entries=150, sequenceid=242, filesize=30.4 K 2024-12-09T06:18:11,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/c4d133afc9dd48c8a679bb10a85ec4a8 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c4d133afc9dd48c8a679bb10a85ec4a8 2024-12-09T06:18:11,201 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c4d133afc9dd48c8a679bb10a85ec4a8, entries=150, sequenceid=242, filesize=11.9 K 2024-12-09T06:18:11,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/74d37ef72a964b1bae9c3edd768cf54d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/74d37ef72a964b1bae9c3edd768cf54d 2024-12-09T06:18:11,205 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/74d37ef72a964b1bae9c3edd768cf54d, entries=150, sequenceid=242, filesize=11.9 K 2024-12-09T06:18:11,205 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=120.76 KB/123660 for 86ed6927f9c6713f75d4b2e28d186f5a in 1246ms, sequenceid=242, compaction requested=true 2024-12-09T06:18:11,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-09T06:18:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-09T06:18:11,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-09T06:18:11,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5380 sec 2024-12-09T06:18:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:11,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=127.47 KB heapSize=365.53 KB 2024-12-09T06:18:11,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:11,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:11,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:11,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:11,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:11,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:11,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.5410 sec 2024-12-09T06:18:11,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120994aeac43d75d4bb7aec834ca5176ad23_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725091207/Put/seqid=0 2024-12-09T06:18:11,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742423_1599 (size=14994) 2024-12-09T06:18:11,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725151243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725151257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725151356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725151466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725151566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,623 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:11,626 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120994aeac43d75d4bb7aec834ca5176ad23_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120994aeac43d75d4bb7aec834ca5176ad23_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:11,627 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6428c61879304034a0a01f93e811ce9c, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:11,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6428c61879304034a0a01f93e811ce9c is 175, key is test_row_0/A:col10/1733725091207/Put/seqid=0 2024-12-09T06:18:11,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742424_1600 (size=39949) 2024-12-09T06:18:11,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725151737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,739 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725151776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T06:18:11,837 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:11,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:18:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-09T06:18:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T06:18:11,840 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:18:11,842 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:18:11,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:18:11,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725151876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T06:18:11,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T06:18:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:11,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:12,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6428c61879304034a0a01f93e811ce9c 2024-12-09T06:18:12,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/9e2957d7c8fb4f9cbee3a1715e2ea07b is 50, key is test_row_0/B:col10/1733725091207/Put/seqid=0 2024-12-09T06:18:12,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742425_1601 (size=12301) 2024-12-09T06:18:12,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/9e2957d7c8fb4f9cbee3a1715e2ea07b 2024-12-09T06:18:12,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/6aa51ba41357401885f4dc2ae51842d4 is 50, key is test_row_0/C:col10/1733725091207/Put/seqid=0 2024-12-09T06:18:12,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742426_1602 (size=12301) 2024-12-09T06:18:12,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/6aa51ba41357401885f4dc2ae51842d4 2024-12-09T06:18:12,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/6428c61879304034a0a01f93e811ce9c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6428c61879304034a0a01f93e811ce9c 2024-12-09T06:18:12,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6428c61879304034a0a01f93e811ce9c, entries=200, sequenceid=364, filesize=39.0 K 2024-12-09T06:18:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/9e2957d7c8fb4f9cbee3a1715e2ea07b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/9e2957d7c8fb4f9cbee3a1715e2ea07b 2024-12-09T06:18:12,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/9e2957d7c8fb4f9cbee3a1715e2ea07b, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T06:18:12,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/6aa51ba41357401885f4dc2ae51842d4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/6aa51ba41357401885f4dc2ae51842d4 2024-12-09T06:18:12,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/6aa51ba41357401885f4dc2ae51842d4, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T06:18:12,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.48 KB/374256, currentSize=53.67 KB/54960 for 86ed6927f9c6713f75d4b2e28d186f5a in 872ms, sequenceid=364, compaction requested=true 2024-12-09T06:18:12,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:12,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:18:12,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:12,080 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:12,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:18:12,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:12,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:18:12,080 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:12,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:12,081 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:12,081 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133640 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:12,081 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/B is initiating minor compaction (all files) 2024-12-09T06:18:12,081 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/A is initiating minor compaction (all files) 2024-12-09T06:18:12,081 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/A in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:12,081 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/B in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:12,081 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6d4fb84b4c6e40609c8df63a450e35f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/960ac0de74a14790b519da50f1bd47cd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/21f742a64b5f4febaba208090a36151e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6428c61879304034a0a01f93e811ce9c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=130.5 K 2024-12-09T06:18:12,081 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/ce2cc94b275a4e34b32c4f5cd641e88f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/b4cb39d8e3814b20b7de41f316e0eaac, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c4d133afc9dd48c8a679bb10a85ec4a8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/9e2957d7c8fb4f9cbee3a1715e2ea07b] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=48.0 K 2024-12-09T06:18:12,081 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:12,081 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6d4fb84b4c6e40609c8df63a450e35f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/960ac0de74a14790b519da50f1bd47cd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/21f742a64b5f4febaba208090a36151e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6428c61879304034a0a01f93e811ce9c] 2024-12-09T06:18:12,081 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2cc94b275a4e34b32c4f5cd641e88f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733725087353 2024-12-09T06:18:12,081 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d4fb84b4c6e40609c8df63a450e35f3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733725087353 2024-12-09T06:18:12,082 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 960ac0de74a14790b519da50f1bd47cd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733725087501 2024-12-09T06:18:12,082 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting b4cb39d8e3814b20b7de41f316e0eaac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733725087501 2024-12-09T06:18:12,082 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21f742a64b5f4febaba208090a36151e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733725088685 2024-12-09T06:18:12,082 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c4d133afc9dd48c8a679bb10a85ec4a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733725088685 2024-12-09T06:18:12,082 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6428c61879304034a0a01f93e811ce9c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733725091181 2024-12-09T06:18:12,082 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e2957d7c8fb4f9cbee3a1715e2ea07b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733725091195 2024-12-09T06:18:12,087 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:12,088 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:12,089 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/facfab6d1a9e4dfc8bbaf4ab731d1cf6 is 50, key is test_row_0/B:col10/1733725091207/Put/seqid=0 2024-12-09T06:18:12,089 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120977ae6e89df234462b93a700dbe95d24e_86ed6927f9c6713f75d4b2e28d186f5a store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:12,091 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120977ae6e89df234462b93a700dbe95d24e_86ed6927f9c6713f75d4b2e28d186f5a, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:12,091 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120977ae6e89df234462b93a700dbe95d24e_86ed6927f9c6713f75d4b2e28d186f5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:12,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742427_1603 (size=12813) 2024-12-09T06:18:12,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742428_1604 (size=4469) 2024-12-09T06:18:12,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T06:18:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:12,146 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:18:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:12,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d1f4389371cb4c48affd0b0aa26b7ada_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725091232/Put/seqid=0 2024-12-09T06:18:12,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742429_1605 (size=12454) 2024-12-09T06:18:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T06:18:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:12,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:12,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725092312, max timestamp=1733725092325Num uniques 859; 2024-12-09T06:18:12,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725092312, max timestamp=1733725092325Num uniques 859; 2024-12-09T06:18:12,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:18:12,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:18:12,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733725092312, max timestamp=1733725092325Num uniques 859; 2024-12-09T06:18:12,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:18:12,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725152354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725152355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725152357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725152386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T06:18:12,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725152467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725152467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725152467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,498 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/facfab6d1a9e4dfc8bbaf4ab731d1cf6 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/facfab6d1a9e4dfc8bbaf4ab731d1cf6 2024-12-09T06:18:12,499 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:12,500 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/7ff51adf94e046079a4e7b4666a30bac is 175, key is test_row_0/A:col10/1733725091207/Put/seqid=0 2024-12-09T06:18:12,504 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/B of 86ed6927f9c6713f75d4b2e28d186f5a into facfab6d1a9e4dfc8bbaf4ab731d1cf6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:12,504 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:12,504 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/B, priority=12, startTime=1733725092080; duration=0sec 2024-12-09T06:18:12,504 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:12,504 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:B 2024-12-09T06:18:12,505 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T06:18:12,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742430_1606 (size=31767) 2024-12-09T06:18:12,507 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T06:18:12,508 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/C is initiating minor compaction (all files) 2024-12-09T06:18:12,508 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/C in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:12,508 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/877c5d171d2046c9aec59f7b30e75f69, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/4cd38a059dd04756a8e7a764b637962f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/74d37ef72a964b1bae9c3edd768cf54d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/6aa51ba41357401885f4dc2ae51842d4] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=48.0 K 2024-12-09T06:18:12,508 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 877c5d171d2046c9aec59f7b30e75f69, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733725087353 2024-12-09T06:18:12,509 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd38a059dd04756a8e7a764b637962f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733725087501 2024-12-09T06:18:12,509 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d37ef72a964b1bae9c3edd768cf54d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733725088685 2024-12-09T06:18:12,510 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa51ba41357401885f4dc2ae51842d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733725091195 2024-12-09T06:18:12,511 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/7ff51adf94e046079a4e7b4666a30bac as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7ff51adf94e046079a4e7b4666a30bac 2024-12-09T06:18:12,524 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/A of 86ed6927f9c6713f75d4b2e28d186f5a into 7ff51adf94e046079a4e7b4666a30bac(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:12,524 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:12,524 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/A, priority=12, startTime=1733725092080; duration=0sec 2024-12-09T06:18:12,524 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:12,524 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:A 2024-12-09T06:18:12,525 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:12,525 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/bfe359725058441d85c2e956f34fa83a is 50, key is test_row_0/C:col10/1733725091207/Put/seqid=0 2024-12-09T06:18:12,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742431_1607 (size=12813) 2024-12-09T06:18:12,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:12,560 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d1f4389371cb4c48affd0b0aa26b7ada_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d1f4389371cb4c48affd0b0aa26b7ada_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:12,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/f4c1d2029688434697ad5cf1c868621c, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:12,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/f4c1d2029688434697ad5cf1c868621c is 175, key is test_row_0/A:col10/1733725091232/Put/seqid=0 2024-12-09T06:18:12,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742432_1608 (size=31255) 2024-12-09T06:18:12,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725152676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725152676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725152676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,937 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/bfe359725058441d85c2e956f34fa83a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/bfe359725058441d85c2e956f34fa83a 2024-12-09T06:18:12,940 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/C of 86ed6927f9c6713f75d4b2e28d186f5a into bfe359725058441d85c2e956f34fa83a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:12,940 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:12,940 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/C, priority=12, startTime=1733725092080; duration=0sec 2024-12-09T06:18:12,940 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:12,940 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:C 2024-12-09T06:18:12,971 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/f4c1d2029688434697ad5cf1c868621c 2024-12-09T06:18:12,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/c978e04141eb4d87ba7610048908bf14 is 50, key is test_row_0/B:col10/1733725091232/Put/seqid=0 2024-12-09T06:18:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T06:18:12,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742433_1609 (size=12301) 2024-12-09T06:18:12,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725152986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725152986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725152986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:13,380 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/c978e04141eb4d87ba7610048908bf14 2024-12-09T06:18:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/3bb498a429bb4f77b5ed1d2079657fc4 is 50, key is test_row_0/C:col10/1733725091232/Put/seqid=0 2024-12-09T06:18:13,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742434_1610 (size=12301) 2024-12-09T06:18:13,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725153407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:13,409 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725153497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725153497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725153497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:13,802 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/3bb498a429bb4f77b5ed1d2079657fc4 2024-12-09T06:18:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/f4c1d2029688434697ad5cf1c868621c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/f4c1d2029688434697ad5cf1c868621c 2024-12-09T06:18:13,810 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/f4c1d2029688434697ad5cf1c868621c, entries=150, sequenceid=375, filesize=30.5 K 2024-12-09T06:18:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/c978e04141eb4d87ba7610048908bf14 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c978e04141eb4d87ba7610048908bf14 2024-12-09T06:18:13,813 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c978e04141eb4d87ba7610048908bf14, entries=150, sequenceid=375, filesize=12.0 K 2024-12-09T06:18:13,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/3bb498a429bb4f77b5ed1d2079657fc4 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3bb498a429bb4f77b5ed1d2079657fc4 2024-12-09T06:18:13,816 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3bb498a429bb4f77b5ed1d2079657fc4, entries=150, sequenceid=375, filesize=12.0 K 2024-12-09T06:18:13,817 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 86ed6927f9c6713f75d4b2e28d186f5a in 1671ms, sequenceid=375, compaction requested=false 2024-12-09T06:18:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-09T06:18:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-09T06:18:13,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-09T06:18:13,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9760 sec 2024-12-09T06:18:13,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9820 sec 2024-12-09T06:18:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T06:18:13,987 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:13,988 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:18:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-09T06:18:13,989 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:18:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T06:18:13,991 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:18:13,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:18:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T06:18:14,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-09T06:18:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:14,143 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T06:18:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:14,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a24bfa359bc84a85b1d4be3ffd69715b_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725092354/Put/seqid=0 2024-12-09T06:18:14,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742435_1611 (size=12454) 2024-12-09T06:18:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T06:18:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:14,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725154513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725154516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725154517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:14,565 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a24bfa359bc84a85b1d4be3ffd69715b_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a24bfa359bc84a85b1d4be3ffd69715b_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/4d94e93304564f5f9c4c1a8acb5861ab, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:14,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/4d94e93304564f5f9c4c1a8acb5861ab is 175, key is test_row_0/A:col10/1733725092354/Put/seqid=0 2024-12-09T06:18:14,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742436_1612 (size=31255) 2024-12-09T06:18:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T06:18:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725154626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725154626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725154626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725154837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725154837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725154837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:14,969 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/4d94e93304564f5f9c4c1a8acb5861ab 2024-12-09T06:18:14,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/1f58e95fe2fd433b96542a4bf19a4fd1 is 50, key is test_row_0/B:col10/1733725092354/Put/seqid=0 2024-12-09T06:18:14,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742437_1613 (size=12301) 2024-12-09T06:18:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T06:18:15,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725155146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:15,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:15,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725155146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725155147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:15,377 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/1f58e95fe2fd433b96542a4bf19a4fd1 2024-12-09T06:18:15,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/e47e006f29c94902bbaa63287bdb6bc1 is 50, key is test_row_0/C:col10/1733725092354/Put/seqid=0 2024-12-09T06:18:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742438_1614 (size=12301) 2024-12-09T06:18:15,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56396 deadline: 1733725155427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:15,429 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_2' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:15,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725155657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:15,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725155657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:15,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725155657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:15,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56428 deadline: 1733725155757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:15,759 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 82ac79f0323b,42797,1733724930590 for 'test_row_1' in 86ed6927f9c6713f75d4b2e28d186f5a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-09T06:18:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:15,786 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/e47e006f29c94902bbaa63287bdb6bc1 2024-12-09T06:18:15,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/4d94e93304564f5f9c4c1a8acb5861ab as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4d94e93304564f5f9c4c1a8acb5861ab 2024-12-09T06:18:15,793 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4d94e93304564f5f9c4c1a8acb5861ab, entries=150, sequenceid=413, filesize=30.5 K 2024-12-09T06:18:15,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/1f58e95fe2fd433b96542a4bf19a4fd1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/1f58e95fe2fd433b96542a4bf19a4fd1 2024-12-09T06:18:15,797 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/1f58e95fe2fd433b96542a4bf19a4fd1, entries=150, sequenceid=413, filesize=12.0 K 2024-12-09T06:18:15,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/e47e006f29c94902bbaa63287bdb6bc1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e47e006f29c94902bbaa63287bdb6bc1 2024-12-09T06:18:15,802 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e47e006f29c94902bbaa63287bdb6bc1, entries=150, sequenceid=413, filesize=12.0 K 2024-12-09T06:18:15,804 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 86ed6927f9c6713f75d4b2e28d186f5a in 1661ms, sequenceid=413, compaction requested=true 2024-12-09T06:18:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-09T06:18:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-09T06:18:15,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-09T06:18:15,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-12-09T06:18:15,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8200 sec 2024-12-09T06:18:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T06:18:16,137 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:16,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:18:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-09T06:18:16,139 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:18:16,140 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:18:16,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:18:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T06:18:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T06:18:16,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T06:18:16,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:16,293 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T06:18:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:16,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c7163cefb39f436d8c6764c1482e9ee2_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725094516/Put/seqid=0 2024-12-09T06:18:16,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742439_1615 (size=12454) 2024-12-09T06:18:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T06:18:16,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:16,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:16,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733725096690, max timestamp=1733725096703Num uniques 859; 2024-12-09T06:18:16,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T06:18:16,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725096695, max timestamp=1733725096703Num uniques 859; 2024-12-09T06:18:16,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:16,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725096695, max timestamp=1733725096703Num uniques 859; 2024-12-09T06:18:16,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:16,712 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c7163cefb39f436d8c6764c1482e9ee2_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7163cefb39f436d8c6764c1482e9ee2_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:16,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/687706c8a6084852872eaebc8593ca06, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:16,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/687706c8a6084852872eaebc8593ca06 is 175, key is test_row_0/A:col10/1733725094516/Put/seqid=0 2024-12-09T06:18:16,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742440_1616 (size=31255) 2024-12-09T06:18:16,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725156739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725156740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:16,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725156740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T06:18:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725156847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725156847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:16,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725156847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:17,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56370 deadline: 1733725157059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56416 deadline: 1733725157061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T06:18:17,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42797 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56372 deadline: 1733725157061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 , the old value is region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T06:18:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., hostname=82ac79f0323b,42797,1733724930590, seqNum=5 because the exception is null or not the one we care about 2024-12-09T06:18:17,128 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=424, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/687706c8a6084852872eaebc8593ca06 2024-12-09T06:18:17,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/4a536fec43ec434f90ee510aca084906 is 50, key is test_row_0/B:col10/1733725094516/Put/seqid=0 2024-12-09T06:18:17,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742441_1617 (size=12301) 2024-12-09T06:18:17,177 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/4a536fec43ec434f90ee510aca084906 2024-12-09T06:18:17,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/8c0dbfbe42f94a1ea1c8e0a39343c606 is 50, key is test_row_0/C:col10/1733725094516/Put/seqid=0 2024-12-09T06:18:17,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742442_1618 (size=12301) 2024-12-09T06:18:17,234 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/8c0dbfbe42f94a1ea1c8e0a39343c606 2024-12-09T06:18:17,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/687706c8a6084852872eaebc8593ca06 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/687706c8a6084852872eaebc8593ca06 2024-12-09T06:18:17,245 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/687706c8a6084852872eaebc8593ca06, entries=150, sequenceid=424, filesize=30.5 K 2024-12-09T06:18:17,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/4a536fec43ec434f90ee510aca084906 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4a536fec43ec434f90ee510aca084906 2024-12-09T06:18:17,251 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4a536fec43ec434f90ee510aca084906, entries=150, sequenceid=424, filesize=12.0 K 2024-12-09T06:18:17,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/8c0dbfbe42f94a1ea1c8e0a39343c606 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8c0dbfbe42f94a1ea1c8e0a39343c606 2024-12-09T06:18:17,267 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8c0dbfbe42f94a1ea1c8e0a39343c606, entries=150, sequenceid=424, filesize=12.0 K 2024-12-09T06:18:17,270 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for 86ed6927f9c6713f75d4b2e28d186f5a in 976ms, sequenceid=424, compaction requested=true 2024-12-09T06:18:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-09T06:18:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-09T06:18:17,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-09T06:18:17,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1310 sec 2024-12-09T06:18:17,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.1370 sec 2024-12-09T06:18:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T06:18:17,276 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:17,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-09T06:18:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-09T06:18:17,279 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T06:18:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T06:18:17,280 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T06:18:17,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T06:18:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:17,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725096731, max timestamp=1733725097378Num uniques 1068; 2024-12-09T06:18:17,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725096731, max timestamp=1733725097378Num uniques 1018; 2024-12-09T06:18:17,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-09T06:18:17,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-12-09T06:18:17,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725096731, max timestamp=1733725097378Num uniques 1018; 2024-12-09T06:18:17,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-09T06:18:17,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=24.44 KB heapSize=59.91 KB 2024-12-09T06:18:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T06:18:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:17,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093022f6a42bd94854b67002056dc5dcdc_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725097379/Put/seqid=0 2024-12-09T06:18:17,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742443_1619 (size=12454) 2024-12-09T06:18:17,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T06:18:17,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:17,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725097447, max timestamp=1733725097477Num uniques 859; 2024-12-09T06:18:17,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:17,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733725097447, max timestamp=1733725097477Num uniques 859; 2024-12-09T06:18:17,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T06:18:17,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733725097447, max timestamp=1733725097477Num uniques 859; 2024-12-09T06:18:17,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T06:18:17,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733725097537, max timestamp=1733725097549Num uniques 1018; 2024-12-09T06:18:17,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:18:17,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1733725097537, max timestamp=1733725097551Num uniques 1018; 2024-12-09T06:18:17,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=7680, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T06:18:17,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733725097537, max timestamp=1733725097549Num uniques 1009; 2024-12-09T06:18:17,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T06:18:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T06:18:17,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T06:18:17,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:17,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733725097607, max timestamp=1733725097626Num uniques 1027; 2024-12-09T06:18:17,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T06:18:17,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733725097607, max timestamp=1733725097626Num uniques 1027; 2024-12-09T06:18:17,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733725097607, max timestamp=1733725097626Num uniques 1027; 2024-12-09T06:18:17,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T06:18:17,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T06:18:17,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733725097673, max timestamp=1733725097686Num uniques 1036; 2024-12-09T06:18:17,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T06:18:17,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733725097673, max timestamp=1733725097686Num uniques 1036; 2024-12-09T06:18:17,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T06:18:17,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733725097673, max timestamp=1733725097686Num uniques 1036; 2024-12-09T06:18:17,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T06:18:17,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T06:18:17,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:17,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733725097730, max timestamp=1733725097754Num uniques 1045; 2024-12-09T06:18:17,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T06:18:17,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733725097730, max timestamp=1733725097754Num uniques 1045; 2024-12-09T06:18:17,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T06:18:17,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733725097730, max timestamp=1733725097757Num uniques 1045; 2024-12-09T06:18:17,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T06:18:17,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:17,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733725097788, max timestamp=1733725097834Num uniques 1054; 2024-12-09T06:18:17,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T06:18:17,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733725097788, max timestamp=1733725097831Num uniques 1054; 2024-12-09T06:18:17,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T06:18:17,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733725097788, max timestamp=1733725097834Num uniques 1054; 2024-12-09T06:18:17,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T06:18:17,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093022f6a42bd94854b67002056dc5dcdc_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093022f6a42bd94854b67002056dc5dcdc_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:17,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/8da53a5c99934cefa359c49c867702ad, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:17,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/8da53a5c99934cefa359c49c867702ad is 175, key is test_row_0/A:col10/1733725097379/Put/seqid=0 2024-12-09T06:18:17,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742444_1620 (size=31255) 2024-12-09T06:18:17,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=10.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/8da53a5c99934cefa359c49c867702ad 2024-12-09T06:18:17,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/539fd360ea3c4142a25452422caa8a7d is 50, key is test_row_0/B:col10/1733725097379/Put/seqid=0 2024-12-09T06:18:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742445_1621 (size=12301) 2024-12-09T06:18:17,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/539fd360ea3c4142a25452422caa8a7d 2024-12-09T06:18:17,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/5b60cbcf2638469cbd3d97f1dd9e518c is 50, key is test_row_0/C:col10/1733725097379/Put/seqid=0 2024-12-09T06:18:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733725097881, max timestamp=1733725097900Num uniques 1063; 2024-12-09T06:18:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-09T06:18:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733725097881, max timestamp=1733725097900Num uniques 1063; 2024-12-09T06:18:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-09T06:18:17,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7457, totalHeapSize=17192, min timestamp=1733725097881, max timestamp=1733725097900Num uniques 1063; 2024-12-09T06:18:17,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48677, new segment data size=7457, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-09T06:18:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T06:18:17,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T06:18:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:17,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:17,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742446_1622 (size=12301) 2024-12-09T06:18:17,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733725097927, max timestamp=1733725097958Num uniques 1022; 2024-12-09T06:18:17,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-09T06:18:17,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733725097927, max timestamp=1733725097958Num uniques 1022; 2024-12-09T06:18:17,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-09T06:18:17,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733725097927, max timestamp=1733725097958Num uniques 1022; 2024-12-09T06:18:17,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-09T06:18:18,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733725097986, max timestamp=1733725098021Num uniques 1081; 2024-12-09T06:18:18,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-09T06:18:18,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733725097986, max timestamp=1733725098021Num uniques 1031; 2024-12-09T06:18:18,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T06:18:18,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733725097986, max timestamp=1733725098021Num uniques 1031; 2024-12-09T06:18:18,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T06:18:18,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733725098028, max timestamp=1733725098060Num uniques 1090; 2024-12-09T06:18:18,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T06:18:18,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733725098028, max timestamp=1733725098060Num uniques 1090; 2024-12-09T06:18:18,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T06:18:18,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733725098028, max timestamp=1733725098060Num uniques 1090; 2024-12-09T06:18:18,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/82ac79f0323b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T06:18:18,065 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-12-09T06:18:18,065 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:18,065 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,065 INFO [Thread-2417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2417. 2024-12-09T06:18:18,065 DEBUG [Thread-2417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:18,065 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,065 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,065 INFO [Thread-2426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2426. 2024-12-09T06:18:18,065 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,065 DEBUG [Thread-2426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:18,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:18,065 DEBUG [Thread-2426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:18,065 DEBUG [Thread-2426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:18,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T06:18:18,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:18,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:18,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:18,067 INFO [Thread-2415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2415. 2024-12-09T06:18:18,067 DEBUG [Thread-2415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:18,067 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:18,067 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,067 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:18,069 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-12-09T06:18:18,069 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-12-09T06:18:18,069 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:18,069 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,069 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:18,069 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,069 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,069 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:18,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:18,070 INFO [Thread-2424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2424. 2024-12-09T06:18:18,070 DEBUG [Thread-2424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:18,070 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,070 DEBUG [Thread-2424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:18,070 INFO [Thread-2428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2428. 2024-12-09T06:18:18,070 DEBUG [Thread-2428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:18,070 DEBUG [Thread-2428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,071 DEBUG [Thread-2428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:18,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:18,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T06:18:18,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:18,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T06:18:18,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/5b60cbcf2638469cbd3d97f1dd9e518c 2024-12-09T06:18:18,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/8da53a5c99934cefa359c49c867702ad as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/8da53a5c99934cefa359c49c867702ad 2024-12-09T06:18:18,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/8da53a5c99934cefa359c49c867702ad, entries=150, sequenceid=463, filesize=30.5 K 2024-12-09T06:18:18,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/539fd360ea3c4142a25452422caa8a7d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/539fd360ea3c4142a25452422caa8a7d 2024-12-09T06:18:18,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/539fd360ea3c4142a25452422caa8a7d, entries=150, sequenceid=463, filesize=12.0 K 2024-12-09T06:18:18,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/5b60cbcf2638469cbd3d97f1dd9e518c as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/5b60cbcf2638469cbd3d97f1dd9e518c 2024-12-09T06:18:18,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/5b60cbcf2638469cbd3d97f1dd9e518c, entries=150, sequenceid=463, filesize=12.0 K 2024-12-09T06:18:18,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~31.15 KB/31896, heapSize ~79.78 KB/81696, currentSize=60.38 KB/61830 for 86ed6927f9c6713f75d4b2e28d186f5a in 954ms, sequenceid=463, compaction requested=true 2024-12-09T06:18:18,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:18,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T06:18:18,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:18,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T06:18:18,339 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T06:18:18,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:18,339 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T06:18:18,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86ed6927f9c6713f75d4b2e28d186f5a:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T06:18:18,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62017 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156787 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/A is initiating minor compaction (all files) 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/B is initiating minor compaction (all files) 2024-12-09T06:18:18,340 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/A in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,340 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/B in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,340 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7ff51adf94e046079a4e7b4666a30bac, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/f4c1d2029688434697ad5cf1c868621c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4d94e93304564f5f9c4c1a8acb5861ab, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/687706c8a6084852872eaebc8593ca06, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/8da53a5c99934cefa359c49c867702ad] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=153.1 K 2024-12-09T06:18:18,340 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/facfab6d1a9e4dfc8bbaf4ab731d1cf6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c978e04141eb4d87ba7610048908bf14, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/1f58e95fe2fd433b96542a4bf19a4fd1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4a536fec43ec434f90ee510aca084906, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/539fd360ea3c4142a25452422caa8a7d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=60.6 K 2024-12-09T06:18:18,340 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. files: [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7ff51adf94e046079a4e7b4666a30bac, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/f4c1d2029688434697ad5cf1c868621c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4d94e93304564f5f9c4c1a8acb5861ab, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/687706c8a6084852872eaebc8593ca06, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/8da53a5c99934cefa359c49c867702ad] 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting facfab6d1a9e4dfc8bbaf4ab731d1cf6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733725091195 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ff51adf94e046079a4e7b4666a30bac, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733725091195 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting c978e04141eb4d87ba7610048908bf14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733725091211 2024-12-09T06:18:18,340 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c1d2029688434697ad5cf1c868621c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733725091211 2024-12-09T06:18:18,341 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f58e95fe2fd433b96542a4bf19a4fd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733725092351 2024-12-09T06:18:18,341 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d94e93304564f5f9c4c1a8acb5861ab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733725092351 2024-12-09T06:18:18,341 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 687706c8a6084852872eaebc8593ca06, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733725094509 2024-12-09T06:18:18,341 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a536fec43ec434f90ee510aca084906, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733725094509 2024-12-09T06:18:18,341 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8da53a5c99934cefa359c49c867702ad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733725096738 2024-12-09T06:18:18,341 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 539fd360ea3c4142a25452422caa8a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733725096738 2024-12-09T06:18:18,350 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:18,352 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120968daf11c48744d319a492a04a0a30e62_86ed6927f9c6713f75d4b2e28d186f5a store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:18,353 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#B#compaction#529 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:18,354 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/4f95184321914beba76dd5f62f0ee57d is 50, key is test_row_0/B:col10/1733725097379/Put/seqid=0 2024-12-09T06:18:18,357 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120968daf11c48744d319a492a04a0a30e62_86ed6927f9c6713f75d4b2e28d186f5a, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:18,357 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120968daf11c48744d319a492a04a0a30e62_86ed6927f9c6713f75d4b2e28d186f5a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:18,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742447_1623 (size=12983) 2024-12-09T06:18:18,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742448_1624 (size=4469) 2024-12-09T06:18:18,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42797 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T06:18:18,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,372 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=60.38 KB heapSize=158.25 KB 2024-12-09T06:18:18,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:18,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:18,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:18,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:18,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:18,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T06:18:18,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209bed684229d334a539da65a8c215970fd_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_0/A:col10/1733725098067/Put/seqid=0 2024-12-09T06:18:18,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742449_1625 (size=12454) 2024-12-09T06:18:18,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:18,407 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209bed684229d334a539da65a8c215970fd_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bed684229d334a539da65a8c215970fd_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/06a9d8ee660f4c6a813bf72b1522c829, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:18,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/06a9d8ee660f4c6a813bf72b1522c829 is 175, key is test_row_0/A:col10/1733725098067/Put/seqid=0 2024-12-09T06:18:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T06:18:18,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742450_1626 (size=31255) 2024-12-09T06:18:18,770 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/4f95184321914beba76dd5f62f0ee57d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4f95184321914beba76dd5f62f0ee57d 2024-12-09T06:18:18,773 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#A#compaction#528 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:18,774 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/7586e7b2aa2649efb9a955deac2aed3b is 175, key is test_row_0/A:col10/1733725097379/Put/seqid=0 2024-12-09T06:18:18,777 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/B of 86ed6927f9c6713f75d4b2e28d186f5a into 4f95184321914beba76dd5f62f0ee57d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:18,777 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:18,777 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/B, priority=11, startTime=1733725098339; duration=0sec 2024-12-09T06:18:18,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742451_1627 (size=31937) 2024-12-09T06:18:18,777 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T06:18:18,777 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:B 2024-12-09T06:18:18,777 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T06:18:18,778 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62017 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T06:18:18,778 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1541): 86ed6927f9c6713f75d4b2e28d186f5a/C is initiating minor compaction (all files) 2024-12-09T06:18:18,778 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86ed6927f9c6713f75d4b2e28d186f5a/C in TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:18,779 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/bfe359725058441d85c2e956f34fa83a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3bb498a429bb4f77b5ed1d2079657fc4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e47e006f29c94902bbaa63287bdb6bc1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8c0dbfbe42f94a1ea1c8e0a39343c606, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/5b60cbcf2638469cbd3d97f1dd9e518c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp, totalSize=60.6 K 2024-12-09T06:18:18,779 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe359725058441d85c2e956f34fa83a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733725091195 2024-12-09T06:18:18,779 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb498a429bb4f77b5ed1d2079657fc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733725091211 2024-12-09T06:18:18,779 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting e47e006f29c94902bbaa63287bdb6bc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733725092351 2024-12-09T06:18:18,779 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c0dbfbe42f94a1ea1c8e0a39343c606, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733725094509 2024-12-09T06:18:18,780 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b60cbcf2638469cbd3d97f1dd9e518c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733725096738 2024-12-09T06:18:18,789 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86ed6927f9c6713f75d4b2e28d186f5a#C#compaction#531 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T06:18:18,789 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/502c8c9f2c0d41fb8f7d0e861dcb108e is 50, key is test_row_0/C:col10/1733725097379/Put/seqid=0 2024-12-09T06:18:18,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742452_1628 (size=12983) 2024-12-09T06:18:18,819 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=642, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/06a9d8ee660f4c6a813bf72b1522c829 2024-12-09T06:18:18,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/cb5e4ac7ab454d8e8710107fc550db8b is 50, key is test_row_0/B:col10/1733725098067/Put/seqid=0 2024-12-09T06:18:18,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742453_1629 (size=12301) 2024-12-09T06:18:19,182 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/7586e7b2aa2649efb9a955deac2aed3b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7586e7b2aa2649efb9a955deac2aed3b 2024-12-09T06:18:19,186 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/A of 86ed6927f9c6713f75d4b2e28d186f5a into 7586e7b2aa2649efb9a955deac2aed3b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:19,186 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:19,186 INFO [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/A, priority=11, startTime=1733725098339; duration=0sec 2024-12-09T06:18:19,186 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:19,186 DEBUG [RS:0;82ac79f0323b:42797-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:A 2024-12-09T06:18:19,196 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/502c8c9f2c0d41fb8f7d0e861dcb108e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/502c8c9f2c0d41fb8f7d0e861dcb108e 2024-12-09T06:18:19,201 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86ed6927f9c6713f75d4b2e28d186f5a/C of 86ed6927f9c6713f75d4b2e28d186f5a into 502c8c9f2c0d41fb8f7d0e861dcb108e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T06:18:19,201 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:19,201 INFO [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., storeName=86ed6927f9c6713f75d4b2e28d186f5a/C, priority=11, startTime=1733725098339; duration=0sec 2024-12-09T06:18:19,201 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T06:18:19,201 DEBUG [RS:0;82ac79f0323b:42797-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86ed6927f9c6713f75d4b2e28d186f5a:C 2024-12-09T06:18:19,235 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/cb5e4ac7ab454d8e8710107fc550db8b 2024-12-09T06:18:19,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/d600a79885cf4e54965074ae05835f60 is 50, key is test_row_0/C:col10/1733725098067/Put/seqid=0 2024-12-09T06:18:19,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742454_1630 (size=12301) 2024-12-09T06:18:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T06:18:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42797 {}] regionserver.HRegion(8855): Flush requested on 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:19,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. as already flushing 2024-12-09T06:18:19,448 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-12-09T06:18:19,448 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:19,448 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:19,448 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:19,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:19,653 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/d600a79885cf4e54965074ae05835f60 2024-12-09T06:18:19,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/06a9d8ee660f4c6a813bf72b1522c829 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/06a9d8ee660f4c6a813bf72b1522c829 2024-12-09T06:18:19,660 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/06a9d8ee660f4c6a813bf72b1522c829, entries=150, sequenceid=642, filesize=30.5 K 2024-12-09T06:18:19,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/cb5e4ac7ab454d8e8710107fc550db8b as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/cb5e4ac7ab454d8e8710107fc550db8b 2024-12-09T06:18:19,664 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/cb5e4ac7ab454d8e8710107fc550db8b, entries=150, sequenceid=642, filesize=12.0 K 2024-12-09T06:18:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/d600a79885cf4e54965074ae05835f60 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/d600a79885cf4e54965074ae05835f60 2024-12-09T06:18:19,668 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/d600a79885cf4e54965074ae05835f60, entries=150, sequenceid=642, filesize=12.0 K 2024-12-09T06:18:19,669 INFO [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.20 KB/162000, currentSize=6.71 KB/6870 for 86ed6927f9c6713f75d4b2e28d186f5a in 1297ms, sequenceid=642, compaction requested=false 2024-12-09T06:18:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 86ed6927f9c6713f75d4b2e28d186f5a: 2024-12-09T06:18:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/82ac79f0323b:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-09T06:18:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-09T06:18:19,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-09T06:18:19,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3900 sec 2024-12-09T06:18:19,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.3960 sec 2024-12-09T06:18:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T06:18:21,437 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:25,868 INFO [Thread-2413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2413. 2024-12-09T06:18:25,868 DEBUG [Thread-2413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T06:18:25,868 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:25,868 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:25,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 170 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3835 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3944 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3652 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3845 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3910 2024-12-09T06:18:25,868 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T06:18:25,868 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T06:18:25,868 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:18:25,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:25,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:25,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:25,869 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-09T06:18:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T06:18:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T06:18:25,873 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725105873"}]},"ts":"1733725105873"} 2024-12-09T06:18:25,875 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T06:18:25,875 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T06:18:25,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T06:18:25,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, UNASSIGN}] 2024-12-09T06:18:25,878 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, UNASSIGN 2024-12-09T06:18:25,879 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=86ed6927f9c6713f75d4b2e28d186f5a, regionState=CLOSING, regionLocation=82ac79f0323b,42797,1733724930590 2024-12-09T06:18:25,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, UNASSIGN because future has completed 2024-12-09T06:18:25,880 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T06:18:25,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590}] 2024-12-09T06:18:25,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T06:18:26,033 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:26,033 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T06:18:26,033 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 86ed6927f9c6713f75d4b2e28d186f5a, disabling compactions & flushes 2024-12-09T06:18:26,033 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:26,033 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:26,034 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. after waiting 0 ms 2024-12-09T06:18:26,034 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:26,034 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 86ed6927f9c6713f75d4b2e28d186f5a 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-09T06:18:26,034 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=A 2024-12-09T06:18:26,034 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:26,034 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=B 2024-12-09T06:18:26,034 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:26,034 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86ed6927f9c6713f75d4b2e28d186f5a, store=C 2024-12-09T06:18:26,034 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T06:18:26,041 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120907c7dc33c0394eb28241fc612339d25f_86ed6927f9c6713f75d4b2e28d186f5a is 50, key is test_row_1/A:col10/1733725105866/Put/seqid=0 2024-12-09T06:18:26,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742455_1631 (size=9914) 2024-12-09T06:18:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T06:18:26,445 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T06:18:26,448 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120907c7dc33c0394eb28241fc612339d25f_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120907c7dc33c0394eb28241fc612339d25f_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:26,449 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/fa2feedbc9ed4058a7f66c5585f4b625, store: [table=TestAcidGuarantees family=A region=86ed6927f9c6713f75d4b2e28d186f5a] 2024-12-09T06:18:26,449 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/fa2feedbc9ed4058a7f66c5585f4b625 is 175, key is test_row_1/A:col10/1733725105866/Put/seqid=0 2024-12-09T06:18:26,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742456_1632 (size=22561) 2024-12-09T06:18:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T06:18:26,853 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=650, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/fa2feedbc9ed4058a7f66c5585f4b625 2024-12-09T06:18:26,861 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/94cf9e6ce7ad4119be1bc1860bf55dfe is 50, key is test_row_1/B:col10/1733725105866/Put/seqid=0 2024-12-09T06:18:26,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742457_1633 (size=9857) 2024-12-09T06:18:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T06:18:27,265 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/94cf9e6ce7ad4119be1bc1860bf55dfe 2024-12-09T06:18:27,270 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/ea72f451459f4fa994b7e12412e7f62f is 50, key is test_row_1/C:col10/1733725105866/Put/seqid=0 2024-12-09T06:18:27,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742458_1634 (size=9857) 2024-12-09T06:18:27,678 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/ea72f451459f4fa994b7e12412e7f62f 2024-12-09T06:18:27,688 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/A/fa2feedbc9ed4058a7f66c5585f4b625 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fa2feedbc9ed4058a7f66c5585f4b625 2024-12-09T06:18:27,693 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fa2feedbc9ed4058a7f66c5585f4b625, entries=100, sequenceid=650, filesize=22.0 K 2024-12-09T06:18:27,693 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/B/94cf9e6ce7ad4119be1bc1860bf55dfe as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/94cf9e6ce7ad4119be1bc1860bf55dfe 2024-12-09T06:18:27,697 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/94cf9e6ce7ad4119be1bc1860bf55dfe, entries=100, sequenceid=650, filesize=9.6 K 2024-12-09T06:18:27,697 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/.tmp/C/ea72f451459f4fa994b7e12412e7f62f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/ea72f451459f4fa994b7e12412e7f62f 2024-12-09T06:18:27,701 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/ea72f451459f4fa994b7e12412e7f62f, entries=100, sequenceid=650, filesize=9.6 K 2024-12-09T06:18:27,701 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 86ed6927f9c6713f75d4b2e28d186f5a in 1667ms, sequenceid=650, compaction requested=true 2024-12-09T06:18:27,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/89f96684cb3e4ec1812ccd79af5d1ab6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/74f195c0a95f4dfa9418edccad2ac3dd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/421d023c96e94474a0d37cf0d5417ec3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/d56edb9c8021426aa28a7946d59ea90f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/b616fbc700984234ae6206be1e9b3267, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/aa642c0fbd4944c1be747ea18d4c8861, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/2656a60949084f278fc1bfc98a3795de, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fd89fc99072943e19e90f2da2766fadf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4ef52e15e38547a9a22a2760c7f33594, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6a6f6ef3306d4e5fa9bdccdc90170cb4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/0623ee7dae4f4e5abd891be796b6c41e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6d4fb84b4c6e40609c8df63a450e35f3, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/960ac0de74a14790b519da50f1bd47cd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/21f742a64b5f4febaba208090a36151e, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6428c61879304034a0a01f93e811ce9c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7ff51adf94e046079a4e7b4666a30bac, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/f4c1d2029688434697ad5cf1c868621c, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4d94e93304564f5f9c4c1a8acb5861ab, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/687706c8a6084852872eaebc8593ca06, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/8da53a5c99934cefa359c49c867702ad] to archive 2024-12-09T06:18:27,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:18:27,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/89f96684cb3e4ec1812ccd79af5d1ab6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/89f96684cb3e4ec1812ccd79af5d1ab6 2024-12-09T06:18:27,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/74f195c0a95f4dfa9418edccad2ac3dd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/74f195c0a95f4dfa9418edccad2ac3dd 2024-12-09T06:18:27,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/421d023c96e94474a0d37cf0d5417ec3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/421d023c96e94474a0d37cf0d5417ec3 2024-12-09T06:18:27,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/d56edb9c8021426aa28a7946d59ea90f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/d56edb9c8021426aa28a7946d59ea90f 2024-12-09T06:18:27,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/b616fbc700984234ae6206be1e9b3267 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/b616fbc700984234ae6206be1e9b3267 2024-12-09T06:18:27,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/aa642c0fbd4944c1be747ea18d4c8861 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/aa642c0fbd4944c1be747ea18d4c8861 2024-12-09T06:18:27,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/2656a60949084f278fc1bfc98a3795de to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/2656a60949084f278fc1bfc98a3795de 2024-12-09T06:18:27,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fd89fc99072943e19e90f2da2766fadf to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fd89fc99072943e19e90f2da2766fadf 2024-12-09T06:18:27,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4ef52e15e38547a9a22a2760c7f33594 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4ef52e15e38547a9a22a2760c7f33594 2024-12-09T06:18:27,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6a6f6ef3306d4e5fa9bdccdc90170cb4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6a6f6ef3306d4e5fa9bdccdc90170cb4 2024-12-09T06:18:27,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/0623ee7dae4f4e5abd891be796b6c41e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/0623ee7dae4f4e5abd891be796b6c41e 2024-12-09T06:18:27,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6d4fb84b4c6e40609c8df63a450e35f3 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6d4fb84b4c6e40609c8df63a450e35f3 2024-12-09T06:18:27,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/960ac0de74a14790b519da50f1bd47cd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/960ac0de74a14790b519da50f1bd47cd 2024-12-09T06:18:27,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/21f742a64b5f4febaba208090a36151e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/21f742a64b5f4febaba208090a36151e 2024-12-09T06:18:27,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6428c61879304034a0a01f93e811ce9c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/6428c61879304034a0a01f93e811ce9c 2024-12-09T06:18:27,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7ff51adf94e046079a4e7b4666a30bac to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7ff51adf94e046079a4e7b4666a30bac 2024-12-09T06:18:27,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/f4c1d2029688434697ad5cf1c868621c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/f4c1d2029688434697ad5cf1c868621c 2024-12-09T06:18:27,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4d94e93304564f5f9c4c1a8acb5861ab to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/4d94e93304564f5f9c4c1a8acb5861ab 2024-12-09T06:18:27,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/687706c8a6084852872eaebc8593ca06 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/687706c8a6084852872eaebc8593ca06 2024-12-09T06:18:27,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/8da53a5c99934cefa359c49c867702ad to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/8da53a5c99934cefa359c49c867702ad 2024-12-09T06:18:27,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/44028456d11a4ab0a1f0c3393234ff79, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/48c884183c4a492fb3f996f974214c1f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c8faa1b5d225426f95aad0b60bac5c33, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/73dce4d23b7b498ea9c95b47fde26e72, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/672a3cd0dccf4f12b8e35f9270b3bd07, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/8bce83d7ffb140cdbbfc0c9b6530f4b0, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/5e953cfbd4d04c12a18ee52e39c569fc, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/dc6c3fe4841747f183c433538d598e13, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/185ecfd5c6b94f499f9e3b0e77504224, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4270d0a00f3c40a982c5848f034fd3cf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/ce2cc94b275a4e34b32c4f5cd641e88f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/2e33549592484829bcbc20ba87df9894, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/b4cb39d8e3814b20b7de41f316e0eaac, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c4d133afc9dd48c8a679bb10a85ec4a8, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/facfab6d1a9e4dfc8bbaf4ab731d1cf6, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/9e2957d7c8fb4f9cbee3a1715e2ea07b, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c978e04141eb4d87ba7610048908bf14, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/1f58e95fe2fd433b96542a4bf19a4fd1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4a536fec43ec434f90ee510aca084906, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/539fd360ea3c4142a25452422caa8a7d] to archive 2024-12-09T06:18:27,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:18:27,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/44028456d11a4ab0a1f0c3393234ff79 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/44028456d11a4ab0a1f0c3393234ff79 2024-12-09T06:18:27,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/48c884183c4a492fb3f996f974214c1f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/48c884183c4a492fb3f996f974214c1f 2024-12-09T06:18:27,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c8faa1b5d225426f95aad0b60bac5c33 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c8faa1b5d225426f95aad0b60bac5c33 2024-12-09T06:18:27,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/73dce4d23b7b498ea9c95b47fde26e72 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/73dce4d23b7b498ea9c95b47fde26e72 2024-12-09T06:18:27,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/672a3cd0dccf4f12b8e35f9270b3bd07 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/672a3cd0dccf4f12b8e35f9270b3bd07 2024-12-09T06:18:27,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/8bce83d7ffb140cdbbfc0c9b6530f4b0 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/8bce83d7ffb140cdbbfc0c9b6530f4b0 2024-12-09T06:18:27,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/5e953cfbd4d04c12a18ee52e39c569fc to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/5e953cfbd4d04c12a18ee52e39c569fc 2024-12-09T06:18:27,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/dc6c3fe4841747f183c433538d598e13 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/dc6c3fe4841747f183c433538d598e13 2024-12-09T06:18:27,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/185ecfd5c6b94f499f9e3b0e77504224 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/185ecfd5c6b94f499f9e3b0e77504224 2024-12-09T06:18:27,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4270d0a00f3c40a982c5848f034fd3cf to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4270d0a00f3c40a982c5848f034fd3cf 2024-12-09T06:18:27,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/ce2cc94b275a4e34b32c4f5cd641e88f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/ce2cc94b275a4e34b32c4f5cd641e88f 2024-12-09T06:18:27,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/2e33549592484829bcbc20ba87df9894 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/2e33549592484829bcbc20ba87df9894 2024-12-09T06:18:27,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/b4cb39d8e3814b20b7de41f316e0eaac to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/b4cb39d8e3814b20b7de41f316e0eaac 2024-12-09T06:18:27,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c4d133afc9dd48c8a679bb10a85ec4a8 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c4d133afc9dd48c8a679bb10a85ec4a8 2024-12-09T06:18:27,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/facfab6d1a9e4dfc8bbaf4ab731d1cf6 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/facfab6d1a9e4dfc8bbaf4ab731d1cf6 2024-12-09T06:18:27,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/9e2957d7c8fb4f9cbee3a1715e2ea07b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/9e2957d7c8fb4f9cbee3a1715e2ea07b 2024-12-09T06:18:27,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c978e04141eb4d87ba7610048908bf14 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/c978e04141eb4d87ba7610048908bf14 2024-12-09T06:18:27,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/1f58e95fe2fd433b96542a4bf19a4fd1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/1f58e95fe2fd433b96542a4bf19a4fd1 2024-12-09T06:18:27,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4a536fec43ec434f90ee510aca084906 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4a536fec43ec434f90ee510aca084906 2024-12-09T06:18:27,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/539fd360ea3c4142a25452422caa8a7d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/539fd360ea3c4142a25452422caa8a7d 2024-12-09T06:18:27,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c0793431493940b69c71d924d0f53e43, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/348f9bb1af4c4f9f9a3a9db7d7f909a7, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/a5edf02cb8704befb6f7439b41a5a8c4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/79aa3634552642249e6cb530c5d2edba, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e2c912859a2f49009e42f21a15947542, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/41c4fa8cc9ef41239923958200c32f13, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/24bf6498a02e47d79c04696b67cd8a43, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c9c4d6df03f94989a04dd01114c77a29, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3f1feee252b44f7da43c92501e23b6da, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8696176589794aa9b7334c0ff78d4caf, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/877c5d171d2046c9aec59f7b30e75f69, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/9176c98882ee4f68821060c71cb90abd, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/4cd38a059dd04756a8e7a764b637962f, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/74d37ef72a964b1bae9c3edd768cf54d, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/bfe359725058441d85c2e956f34fa83a, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/6aa51ba41357401885f4dc2ae51842d4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3bb498a429bb4f77b5ed1d2079657fc4, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e47e006f29c94902bbaa63287bdb6bc1, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8c0dbfbe42f94a1ea1c8e0a39343c606, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/5b60cbcf2638469cbd3d97f1dd9e518c] to archive 2024-12-09T06:18:27,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T06:18:27,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c0793431493940b69c71d924d0f53e43 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c0793431493940b69c71d924d0f53e43 2024-12-09T06:18:27,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/348f9bb1af4c4f9f9a3a9db7d7f909a7 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/348f9bb1af4c4f9f9a3a9db7d7f909a7 2024-12-09T06:18:27,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/a5edf02cb8704befb6f7439b41a5a8c4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/a5edf02cb8704befb6f7439b41a5a8c4 2024-12-09T06:18:27,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/79aa3634552642249e6cb530c5d2edba to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/79aa3634552642249e6cb530c5d2edba 2024-12-09T06:18:27,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e2c912859a2f49009e42f21a15947542 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e2c912859a2f49009e42f21a15947542 2024-12-09T06:18:27,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/41c4fa8cc9ef41239923958200c32f13 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/41c4fa8cc9ef41239923958200c32f13 2024-12-09T06:18:27,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/24bf6498a02e47d79c04696b67cd8a43 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/24bf6498a02e47d79c04696b67cd8a43 2024-12-09T06:18:27,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c9c4d6df03f94989a04dd01114c77a29 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/c9c4d6df03f94989a04dd01114c77a29 2024-12-09T06:18:27,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3f1feee252b44f7da43c92501e23b6da to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3f1feee252b44f7da43c92501e23b6da 2024-12-09T06:18:27,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8696176589794aa9b7334c0ff78d4caf to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8696176589794aa9b7334c0ff78d4caf 2024-12-09T06:18:27,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/877c5d171d2046c9aec59f7b30e75f69 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/877c5d171d2046c9aec59f7b30e75f69 2024-12-09T06:18:27,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/9176c98882ee4f68821060c71cb90abd to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/9176c98882ee4f68821060c71cb90abd 2024-12-09T06:18:27,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/4cd38a059dd04756a8e7a764b637962f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/4cd38a059dd04756a8e7a764b637962f 2024-12-09T06:18:27,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/74d37ef72a964b1bae9c3edd768cf54d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/74d37ef72a964b1bae9c3edd768cf54d 2024-12-09T06:18:27,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/bfe359725058441d85c2e956f34fa83a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/bfe359725058441d85c2e956f34fa83a 2024-12-09T06:18:27,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/6aa51ba41357401885f4dc2ae51842d4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/6aa51ba41357401885f4dc2ae51842d4 2024-12-09T06:18:27,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3bb498a429bb4f77b5ed1d2079657fc4 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/3bb498a429bb4f77b5ed1d2079657fc4 2024-12-09T06:18:27,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e47e006f29c94902bbaa63287bdb6bc1 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/e47e006f29c94902bbaa63287bdb6bc1 2024-12-09T06:18:27,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8c0dbfbe42f94a1ea1c8e0a39343c606 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/8c0dbfbe42f94a1ea1c8e0a39343c606 2024-12-09T06:18:27,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/5b60cbcf2638469cbd3d97f1dd9e518c to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/5b60cbcf2638469cbd3d97f1dd9e518c 2024-12-09T06:18:27,753 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/recovered.edits/653.seqid, newMaxSeqId=653, maxSeqId=4 2024-12-09T06:18:27,753 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. 2024-12-09T06:18:27,753 DEBUG [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 86ed6927f9c6713f75d4b2e28d186f5a: Waiting for close lock at 1733725106033Running coprocessor pre-close hooks at 1733725106033Disabling compacts and flushes for region at 1733725106033Disabling writes for close at 1733725106034 (+1 ms)Obtaining lock to block concurrent updates at 1733725106034Preparing flush snapshotting stores in 86ed6927f9c6713f75d4b2e28d186f5a at 1733725106034Finished memstore snapshotting TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733725106034Flushing stores of TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a. at 1733725106035 (+1 ms)Flushing 86ed6927f9c6713f75d4b2e28d186f5a/A: creating writer at 1733725106035Flushing mob file 86ed6927f9c6713f75d4b2e28d186f5a/A: appending metadata at 1733725106041 (+6 ms)Flushing mob file 86ed6927f9c6713f75d4b2e28d186f5a/A: closing flushed file at 1733725106041Flushing 86ed6927f9c6713f75d4b2e28d186f5a/A: appending metadata at 1733725106449 (+408 ms)Flushing 86ed6927f9c6713f75d4b2e28d186f5a/A: closing flushed file at 1733725106449Flushing 86ed6927f9c6713f75d4b2e28d186f5a/B: creating writer at 1733725106858 (+409 ms)Flushing 86ed6927f9c6713f75d4b2e28d186f5a/B: appending metadata at 1733725106861 (+3 ms)Flushing 86ed6927f9c6713f75d4b2e28d186f5a/B: closing flushed file at 1733725106861Flushing 86ed6927f9c6713f75d4b2e28d186f5a/C: creating writer at 1733725107268 (+407 ms)Flushing 86ed6927f9c6713f75d4b2e28d186f5a/C: appending metadata at 1733725107270 (+2 ms)Flushing 86ed6927f9c6713f75d4b2e28d186f5a/C: closing flushed file at 1733725107270Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d821606: reopening flushed file at 1733725107686 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@657430eb: reopening flushed file at 1733725107693 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d9c4ef9: reopening flushed file at 1733725107697 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 86ed6927f9c6713f75d4b2e28d186f5a in 1667ms, sequenceid=650, compaction requested=true at 1733725107701 (+4 ms)Writing region close event to WAL at 1733725107750 (+49 ms)Running coprocessor post-close hooks at 1733725107753 (+3 ms)Closed at 1733725107753 2024-12-09T06:18:27,755 INFO [RS_CLOSE_REGION-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:27,755 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=86ed6927f9c6713f75d4b2e28d186f5a, regionState=CLOSED 2024-12-09T06:18:27,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 because future has completed 2024-12-09T06:18:27,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-09T06:18:27,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 86ed6927f9c6713f75d4b2e28d186f5a, server=82ac79f0323b,42797,1733724930590 in 1.8770 sec 2024-12-09T06:18:27,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-12-09T06:18:27,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86ed6927f9c6713f75d4b2e28d186f5a, UNASSIGN in 1.8810 sec 2024-12-09T06:18:27,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-09T06:18:27,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8830 sec 2024-12-09T06:18:27,762 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733725107762"}]},"ts":"1733725107762"} 2024-12-09T06:18:27,763 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T06:18:27,763 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T06:18:27,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8930 sec 2024-12-09T06:18:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T06:18:28,018 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:28,019 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-09T06:18:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:18:28,023 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:18:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T06:18:28,024 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:18:28,026 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,028 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/recovered.edits] 2024-12-09T06:18:28,030 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/06a9d8ee660f4c6a813bf72b1522c829 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/06a9d8ee660f4c6a813bf72b1522c829 2024-12-09T06:18:28,031 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7586e7b2aa2649efb9a955deac2aed3b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/7586e7b2aa2649efb9a955deac2aed3b 2024-12-09T06:18:28,032 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fa2feedbc9ed4058a7f66c5585f4b625 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/A/fa2feedbc9ed4058a7f66c5585f4b625 2024-12-09T06:18:28,033 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4f95184321914beba76dd5f62f0ee57d to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/4f95184321914beba76dd5f62f0ee57d 2024-12-09T06:18:28,034 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/94cf9e6ce7ad4119be1bc1860bf55dfe to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/94cf9e6ce7ad4119be1bc1860bf55dfe 2024-12-09T06:18:28,035 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/cb5e4ac7ab454d8e8710107fc550db8b to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/B/cb5e4ac7ab454d8e8710107fc550db8b 2024-12-09T06:18:28,036 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/502c8c9f2c0d41fb8f7d0e861dcb108e to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/502c8c9f2c0d41fb8f7d0e861dcb108e 2024-12-09T06:18:28,037 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/d600a79885cf4e54965074ae05835f60 to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/d600a79885cf4e54965074ae05835f60 2024-12-09T06:18:28,038 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/ea72f451459f4fa994b7e12412e7f62f to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/C/ea72f451459f4fa994b7e12412e7f62f 2024-12-09T06:18:28,041 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/recovered.edits/653.seqid to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a/recovered.edits/653.seqid 2024-12-09T06:18:28,041 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/default/TestAcidGuarantees/86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,041 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T06:18:28,042 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T06:18:28,042 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T06:18:28,045 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209072f41d95d704d288c1cd22fc1c335ee_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209072f41d95d704d288c1cd22fc1c335ee_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,047 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209075dd9cd57184b10ac0b9b2cfd4e027f_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209075dd9cd57184b10ac0b9b2cfd4e027f_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,048 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120907c7dc33c0394eb28241fc612339d25f_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120907c7dc33c0394eb28241fc612339d25f_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,049 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091790476331134480821b41c90a6118a5_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091790476331134480821b41c90a6118a5_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,050 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093022f6a42bd94854b67002056dc5dcdc_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093022f6a42bd94854b67002056dc5dcdc_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,052 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120949d064b53a95422eac5a32922da71cc1_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120949d064b53a95422eac5a32922da71cc1_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,053 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120949f6eebabf604d62a66cc432c22a8ad3_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120949f6eebabf604d62a66cc432c22a8ad3_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,055 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209507dd3cd3e8f401d9d9b75988ac71ebc_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209507dd3cd3e8f401d9d9b75988ac71ebc_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,056 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095a4d717090de4525adee1df86379f0aa_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095a4d717090de4525adee1df86379f0aa_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,057 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096bed56de1d224939a6022b3a166d24a4_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096bed56de1d224939a6022b3a166d24a4_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,059 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120994aeac43d75d4bb7aec834ca5176ad23_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120994aeac43d75d4bb7aec834ca5176ad23_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,060 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a24bfa359bc84a85b1d4be3ffd69715b_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a24bfa359bc84a85b1d4be3ffd69715b_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,061 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bbf48380045b4c52b7560bb54d4cd061_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bbf48380045b4c52b7560bb54d4cd061_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,062 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bed684229d334a539da65a8c215970fd_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209bed684229d334a539da65a8c215970fd_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,064 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7163cefb39f436d8c6764c1482e9ee2_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c7163cefb39f436d8c6764c1482e9ee2_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,065 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d1f4389371cb4c48affd0b0aa26b7ada_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d1f4389371cb4c48affd0b0aa26b7ada_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,066 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f54ef36d82014531b035fd40bbc9109a_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f54ef36d82014531b035fd40bbc9109a_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,068 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f97e0d60f9d549319ff0d9b86a77e0a2_86ed6927f9c6713f75d4b2e28d186f5a to hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f97e0d60f9d549319ff0d9b86a77e0a2_86ed6927f9c6713f75d4b2e28d186f5a 2024-12-09T06:18:28,068 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T06:18:28,071 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:18:28,073 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T06:18:28,076 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T06:18:28,077 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:18:28,077 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T06:18:28,078 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733725108077"}]},"ts":"9223372036854775807"} 2024-12-09T06:18:28,079 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T06:18:28,079 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 86ed6927f9c6713f75d4b2e28d186f5a, NAME => 'TestAcidGuarantees,,1733725075943.86ed6927f9c6713f75d4b2e28d186f5a.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T06:18:28,080 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T06:18:28,080 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733725108080"}]},"ts":"9223372036854775807"} 2024-12-09T06:18:28,081 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T06:18:28,082 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T06:18:28,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-12-09T06:18:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43241 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T06:18:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T06:18:28,132 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T06:18:28,143 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246 (was 250), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=632 (was 792), ProcessCount=11 (was 11), AvailableMemoryMB=4811 (was 5746) 2024-12-09T06:18:28,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T06:18:28,143 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T06:18:28,143 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:18:28,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:28,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:28,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T06:18:28,144 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T06:18:28,144 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-12-09T06:18:28,144 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=82ac79f0323b,43241,1733724929416 2024-12-09T06:18:28,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T06:18:28,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T06:18:28,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:18:28,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:18:28,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T06:18:28,199 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T06:18:28,199 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:18:28,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:28,200 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '82ac79f0323b,42797,1733724930590' ***** 2024-12-09T06:18:28,200 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T06:18:28,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T06:18:28,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T06:18:28,201 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T06:18:28,202 INFO [RS:0;82ac79f0323b:42797 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T06:18:28,202 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T06:18:28,202 INFO [RS:0;82ac79f0323b:42797 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T06:18:28,202 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(959): stopping server 82ac79f0323b,42797,1733724930590 2024-12-09T06:18:28,202 INFO [RS:0;82ac79f0323b:42797 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T06:18:28,203 INFO [RS:0;82ac79f0323b:42797 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;82ac79f0323b:42797. 2024-12-09T06:18:28,203 DEBUG [RS:0;82ac79f0323b:42797 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T06:18:28,203 DEBUG [RS:0;82ac79f0323b:42797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:28,203 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T06:18:28,203 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T06:18:28,203 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T06:18:28,204 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T06:18:28,205 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T06:18:28,205 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T06:18:28,206 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T06:18:28,206 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T06:18:28,206 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T06:18:28,206 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T06:18:28,206 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T06:18:28,206 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:28,206 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-09T06:18:28,210 INFO [regionserver/82ac79f0323b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T06:18:28,221 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T06:18:28,223 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/info/414db31f09ba4beca24408b8efc3b10e is 95, key is TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8./info:/1733724967374/DeleteFamily/seqid=0 2024-12-09T06:18:28,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742459_1635 (size=6514) 2024-12-09T06:18:28,406 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:28,606 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:28,627 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/info/414db31f09ba4beca24408b8efc3b10e 2024-12-09T06:18:28,645 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/ns/0d99bf303fbf4e55b1f4027c367bb25a is 93, key is TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8./ns:/1733724967319/DeleteFamily/seqid=0 2024-12-09T06:18:28,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742460_1636 (size=5906) 2024-12-09T06:18:28,739 INFO [regionserver/82ac79f0323b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T06:18:28,739 INFO [regionserver/82ac79f0323b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T06:18:28,807 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:29,007 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:29,050 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/ns/0d99bf303fbf4e55b1f4027c367bb25a 2024-12-09T06:18:29,067 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/rep_barrier/58969e333cd74199b9fc09268d3c6ea1 is 102, key is TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8./rep_barrier:/1733724967319/DeleteFamily/seqid=0 2024-12-09T06:18:29,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742461_1637 (size=6025) 2024-12-09T06:18:29,207 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:29,407 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:29,470 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/rep_barrier/58969e333cd74199b9fc09268d3c6ea1 2024-12-09T06:18:29,486 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/table/b436da41db7242c0afeba8c0a5f1be5f is 96, key is TestAcidGuarantees,,1733724939348.89f73a4239a4d6e12de44ad55db0a5e8./table:/1733724967319/DeleteFamily/seqid=0 2024-12-09T06:18:29,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742462_1638 (size=5936) 2024-12-09T06:18:29,607 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:29,808 DEBUG [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T06:18:29,881 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T06:18:29,889 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/table/b436da41db7242c0afeba8c0a5f1be5f 2024-12-09T06:18:29,893 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/info/414db31f09ba4beca24408b8efc3b10e as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/info/414db31f09ba4beca24408b8efc3b10e 2024-12-09T06:18:29,897 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/info/414db31f09ba4beca24408b8efc3b10e, entries=12, sequenceid=90, filesize=6.4 K 2024-12-09T06:18:29,898 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/ns/0d99bf303fbf4e55b1f4027c367bb25a as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/ns/0d99bf303fbf4e55b1f4027c367bb25a 2024-12-09T06:18:29,900 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/ns/0d99bf303fbf4e55b1f4027c367bb25a, entries=8, sequenceid=90, filesize=5.8 K 2024-12-09T06:18:29,901 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/rep_barrier/58969e333cd74199b9fc09268d3c6ea1 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/rep_barrier/58969e333cd74199b9fc09268d3c6ea1 2024-12-09T06:18:29,904 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/rep_barrier/58969e333cd74199b9fc09268d3c6ea1, entries=6, sequenceid=90, filesize=5.9 K 2024-12-09T06:18:29,905 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/.tmp/table/b436da41db7242c0afeba8c0a5f1be5f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/table/b436da41db7242c0afeba8c0a5f1be5f 2024-12-09T06:18:29,909 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/table/b436da41db7242c0afeba8c0a5f1be5f, entries=7, sequenceid=90, filesize=5.8 K 2024-12-09T06:18:29,910 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false 2024-12-09T06:18:29,927 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-09T06:18:29,928 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T06:18:29,928 INFO [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T06:18:29,928 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733725108205Running coprocessor pre-close hooks at 1733725108206 (+1 ms)Disabling compacts and flushes for region at 1733725108206Disabling writes for close at 1733725108206Obtaining lock to block concurrent updates at 1733725108206Preparing flush snapshotting stores in 1588230740 at 1733725108206Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733725108207 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733725108207Flushing 1588230740/info: creating writer at 1733725108208 (+1 ms)Flushing 1588230740/info: appending metadata at 1733725108223 (+15 ms)Flushing 1588230740/info: closing flushed file at 1733725108223Flushing 1588230740/ns: creating writer at 1733725108631 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733725108645 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733725108645Flushing 1588230740/rep_barrier: creating writer at 1733725109053 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733725109066 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733725109066Flushing 1588230740/table: creating writer at 1733725109473 (+407 ms)Flushing 1588230740/table: appending metadata at 1733725109486 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733725109486Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75aab405: reopening flushed file at 1733725109893 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@626a055: reopening flushed file at 1733725109897 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39a69f08: reopening flushed file at 1733725109900 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@487afec9: reopening flushed file at 1733725109904 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false at 1733725109910 (+6 ms)Writing region close event to WAL at 1733725109925 (+15 ms)Running coprocessor post-close hooks at 1733725109928 (+3 ms)Closed at 1733725109928 2024-12-09T06:18:29,928 DEBUG [RS_CLOSE_META-regionserver/82ac79f0323b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T06:18:30,008 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(976): stopping server 82ac79f0323b,42797,1733724930590; all regions closed. 2024-12-09T06:18:30,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741834_1010 (size=25344) 2024-12-09T06:18:30,014 DEBUG [RS:0;82ac79f0323b:42797 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/oldWALs 2024-12-09T06:18:30,014 INFO [RS:0;82ac79f0323b:42797 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 82ac79f0323b%2C42797%2C1733724930590.meta:.meta(num 1733724935807) 2024-12-09T06:18:30,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741833_1009 (size=25163517) 2024-12-09T06:18:30,018 DEBUG [RS:0;82ac79f0323b:42797 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/oldWALs 2024-12-09T06:18:30,018 INFO [RS:0;82ac79f0323b:42797 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 82ac79f0323b%2C42797%2C1733724930590:(num 1733724934947) 2024-12-09T06:18:30,018 DEBUG [RS:0;82ac79f0323b:42797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T06:18:30,019 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T06:18:30,019 INFO [RS:0;82ac79f0323b:42797 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T06:18:30,019 INFO [RS:0;82ac79f0323b:42797 {}] hbase.ChoreService(370): Chore service for: regionserver/82ac79f0323b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T06:18:30,020 INFO [regionserver/82ac79f0323b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T06:18:30,020 INFO [RS:0;82ac79f0323b:42797 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T06:18:30,021 INFO [RS:0;82ac79f0323b:42797 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42797 2024-12-09T06:18:30,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/82ac79f0323b,42797,1733724930590 2024-12-09T06:18:30,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T06:18:30,113 INFO [RS:0;82ac79f0323b:42797 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T06:18:30,114 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [82ac79f0323b,42797,1733724930590] 2024-12-09T06:18:30,176 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/82ac79f0323b,42797,1733724930590 already deleted, retry=false 2024-12-09T06:18:30,177 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 82ac79f0323b,42797,1733724930590 expired; onlineServers=0 2024-12-09T06:18:30,177 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '82ac79f0323b,43241,1733724929416' ***** 2024-12-09T06:18:30,177 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T06:18:30,177 INFO [M:0;82ac79f0323b:43241 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T06:18:30,177 INFO [M:0;82ac79f0323b:43241 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T06:18:30,177 DEBUG [M:0;82ac79f0323b:43241 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T06:18:30,177 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T06:18:30,177 DEBUG [M:0;82ac79f0323b:43241 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T06:18:30,177 DEBUG [master/82ac79f0323b:0:becomeActiveMaster-HFileCleaner.large.0-1733724934367 {}] cleaner.HFileCleaner(306): Exit Thread[master/82ac79f0323b:0:becomeActiveMaster-HFileCleaner.large.0-1733724934367,5,FailOnTimeoutGroup] 2024-12-09T06:18:30,177 INFO [M:0;82ac79f0323b:43241 {}] hbase.ChoreService(370): Chore service for: master/82ac79f0323b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T06:18:30,177 INFO [M:0;82ac79f0323b:43241 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T06:18:30,178 DEBUG [M:0;82ac79f0323b:43241 {}] master.HMaster(1795): Stopping service threads 2024-12-09T06:18:30,178 INFO [M:0;82ac79f0323b:43241 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T06:18:30,178 DEBUG [master/82ac79f0323b:0:becomeActiveMaster-HFileCleaner.small.0-1733724934382 {}] cleaner.HFileCleaner(306): Exit Thread[master/82ac79f0323b:0:becomeActiveMaster-HFileCleaner.small.0-1733724934382,5,FailOnTimeoutGroup] 2024-12-09T06:18:30,178 INFO [M:0;82ac79f0323b:43241 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T06:18:30,178 ERROR [M:0;82ac79f0323b:43241 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:36419 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36419,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-09T06:18:30,178 INFO [M:0;82ac79f0323b:43241 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T06:18:30,179 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T06:18:30,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T06:18:30,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T06:18:30,271 DEBUG [M:0;82ac79f0323b:43241 {}] zookeeper.ZKUtil(347): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T06:18:30,272 WARN [M:0;82ac79f0323b:43241 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T06:18:30,272 INFO [M:0;82ac79f0323b:43241 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/.lastflushedseqids 2024-12-09T06:18:30,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T06:18:30,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42797-0x100093700330001, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T06:18:30,277 INFO [RS:0;82ac79f0323b:42797 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T06:18:30,277 INFO [RS:0;82ac79f0323b:42797 {}] regionserver.HRegionServer(1031): Exiting; stopping=82ac79f0323b,42797,1733724930590; zookeeper connection closed. 2024-12-09T06:18:30,281 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@541836a3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@541836a3 2024-12-09T06:18:30,282 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T06:18:30,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742463_1639 (size=72) 2024-12-09T06:18:30,287 INFO [M:0;82ac79f0323b:43241 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T06:18:30,287 INFO [M:0;82ac79f0323b:43241 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T06:18:30,287 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T06:18:30,287 INFO [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T06:18:30,287 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T06:18:30,287 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T06:18:30,287 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T06:18:30,288 INFO [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=806.23 KB heapSize=990.23 KB 2024-12-09T06:18:30,305 DEBUG [M:0;82ac79f0323b:43241 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/353fc467698142b4989d277b76ab7e8f is 82, key is hbase:meta,,1/info:regioninfo/1733724936058/Put/seqid=0 2024-12-09T06:18:30,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742464_1640 (size=5672) 2024-12-09T06:18:30,710 INFO [M:0;82ac79f0323b:43241 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2284 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/353fc467698142b4989d277b76ab7e8f 2024-12-09T06:18:30,736 DEBUG [M:0;82ac79f0323b:43241 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6bf2dd878b34cccb2db499f2fe8cf0d is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1733725078023/Put/seqid=0 2024-12-09T06:18:30,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742465_1641 (size=45004) 2024-12-09T06:18:31,140 INFO [M:0;82ac79f0323b:43241 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=805.67 KB at sequenceid=2284 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6bf2dd878b34cccb2db499f2fe8cf0d 2024-12-09T06:18:31,144 INFO [M:0;82ac79f0323b:43241 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6bf2dd878b34cccb2db499f2fe8cf0d 2024-12-09T06:18:31,158 DEBUG [M:0;82ac79f0323b:43241 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c2debed00886442aad196ff54e62d882 is 69, key is 82ac79f0323b,42797,1733724930590/rs:state/1733724934433/Put/seqid=0 2024-12-09T06:18:31,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742466_1642 (size=5156) 2024-12-09T06:18:31,170 INFO [M:0;82ac79f0323b:43241 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2284 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c2debed00886442aad196ff54e62d882 2024-12-09T06:18:31,173 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/353fc467698142b4989d277b76ab7e8f as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/353fc467698142b4989d277b76ab7e8f 2024-12-09T06:18:31,176 INFO [M:0;82ac79f0323b:43241 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/353fc467698142b4989d277b76ab7e8f, entries=8, sequenceid=2284, filesize=5.5 K 2024-12-09T06:18:31,176 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6bf2dd878b34cccb2db499f2fe8cf0d as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e6bf2dd878b34cccb2db499f2fe8cf0d 2024-12-09T06:18:31,179 INFO [M:0;82ac79f0323b:43241 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6bf2dd878b34cccb2db499f2fe8cf0d 2024-12-09T06:18:31,179 INFO [M:0;82ac79f0323b:43241 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e6bf2dd878b34cccb2db499f2fe8cf0d, entries=168, sequenceid=2284, filesize=43.9 K 2024-12-09T06:18:31,179 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c2debed00886442aad196ff54e62d882 as hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c2debed00886442aad196ff54e62d882 2024-12-09T06:18:31,181 INFO [M:0;82ac79f0323b:43241 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/0ceb89d2-16e5-e70c-39b2-177fbe39d05d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c2debed00886442aad196ff54e62d882, entries=1, sequenceid=2284, filesize=5.0 K 2024-12-09T06:18:31,182 INFO [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(3140): Finished flush of dataSize ~806.23 KB/825579, heapSize ~989.94 KB/1013696, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 895ms, sequenceid=2284, compaction requested=false 2024-12-09T06:18:31,183 INFO [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T06:18:31,183 DEBUG [M:0;82ac79f0323b:43241 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733725110287Disabling compacts and flushes for region at 1733725110287Disabling writes for close at 1733725110287Obtaining lock to block concurrent updates at 1733725110288 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733725110288Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=825579, getHeapSize=1013936, getOffHeapSize=0, getCellsCount=2477 at 1733725110288Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733725110288Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733725110289 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733725110305 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733725110305Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733725110715 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733725110735 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733725110735Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733725111144 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733725111158 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733725111158Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3367895: reopening flushed file at 1733725111173 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b88ffd2: reopening flushed file at 1733725111176 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a38a3e8: reopening flushed file at 1733725111179 (+3 ms)Finished flush of dataSize ~806.23 KB/825579, heapSize ~989.94 KB/1013696, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 895ms, sequenceid=2284, compaction requested=false at 1733725111182 (+3 ms)Writing region close event to WAL at 1733725111183 (+1 ms)Closed at 1733725111183 2024-12-09T06:18:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741830_1006 (size=975230) 2024-12-09T06:18:31,186 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T06:18:31,186 INFO [M:0;82ac79f0323b:43241 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T06:18:31,186 INFO [M:0;82ac79f0323b:43241 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43241 2024-12-09T06:18:31,186 INFO [M:0;82ac79f0323b:43241 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T06:18:31,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T06:18:31,298 INFO [M:0;82ac79f0323b:43241 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T06:18:31,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43241-0x100093700330000, quorum=127.0.0.1:56248, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T06:18:31,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T06:18:31,310 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T06:18:31,310 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T06:18:31,310 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T06:18:31,310 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/hadoop.log.dir/,STOPPED} 2024-12-09T06:18:31,313 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T06:18:31,313 WARN [BP-519108604-172.17.0.3-1733724896367 heartbeating to localhost/127.0.0.1:36419 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T06:18:31,313 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T06:18:31,313 WARN [BP-519108604-172.17.0.3-1733724896367 heartbeating to localhost/127.0.0.1:36419 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-519108604-172.17.0.3-1733724896367 (Datanode Uuid b99168bb-add6-453c-b13d-a80faae4a1f5) service to localhost/127.0.0.1:36419 2024-12-09T06:18:31,316 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/data/data1/current/BP-519108604-172.17.0.3-1733724896367 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T06:18:31,316 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/cluster_4c732b4f-f425-de59-ed47-9e63bb3d1f50/data/data2/current/BP-519108604-172.17.0.3-1733724896367 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T06:18:31,316 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T06:18:31,322 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T06:18:31,323 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T06:18:31,323 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T06:18:31,323 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T06:18:31,323 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fb8856df-e7c7-2bb7-7b04-486fcd11530b/hadoop.log.dir/,STOPPED} 2024-12-09T06:18:31,339 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T06:18:31,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down